From 971278e7e582c61bd2d555c9f7e335aecc55ee3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ken-H=C3=A5vard=20Lieng?= Date: Tue, 18 Apr 2017 03:02:51 +0200 Subject: [PATCH] Upgrade server dependencies, manage them with govendor --- letsencrypt/letsencrypt.go | 8 +- letsencrypt/user.go | 13 +- letsencrypt/user_test.go | 10 - server/auth.go | 6 +- storage/user_messages.go | 3 +- vendor/github.com/BurntSushi/toml/COMPATIBLE | 3 - vendor/github.com/BurntSushi/toml/COPYING | 14 - vendor/github.com/BurntSushi/toml/Makefile | 19 - vendor/github.com/BurntSushi/toml/README.md | 220 - .../BurntSushi/toml/_examples/example.go | 61 - .../BurntSushi/toml/_examples/hard.toml | 22 - .../BurntSushi/toml/_examples/implicit.toml | 4 - .../toml/_examples/invalid-apples.toml | 6 - .../BurntSushi/toml/_examples/readme1.toml | 5 - .../BurntSushi/toml/_examples/readme2.toml | 1 - .../toml/cmd/toml-test-decoder/COPYING | 14 - .../toml/cmd/toml-test-decoder/README.md | 14 - .../toml/cmd/toml-test-decoder/main.go | 90 - .../toml/cmd/toml-test-encoder/COPYING | 14 - .../toml/cmd/toml-test-encoder/README.md | 14 - .../toml/cmd/toml-test-encoder/main.go | 131 - .../BurntSushi/toml/cmd/tomlv/COPYING | 14 - .../BurntSushi/toml/cmd/tomlv/README.md | 22 - .../BurntSushi/toml/cmd/tomlv/main.go | 61 - vendor/github.com/BurntSushi/toml/decode.go | 493 - .../github.com/BurntSushi/toml/decode_meta.go | 122 - .../github.com/BurntSushi/toml/decode_test.go | 1018 -- vendor/github.com/BurntSushi/toml/doc.go | 27 - vendor/github.com/BurntSushi/toml/encode.go | 562 - .../github.com/BurntSushi/toml/encode_test.go | 566 - .../BurntSushi/toml/encoding_types.go | 19 - .../BurntSushi/toml/encoding_types_1.1.go | 18 - vendor/github.com/BurntSushi/toml/lex.go | 871 -- vendor/github.com/BurntSushi/toml/parse.go | 493 - vendor/github.com/BurntSushi/toml/session.vim | 1 - .../github.com/BurntSushi/toml/type_check.go | 91 - .../github.com/BurntSushi/toml/type_fields.go | 241 - .../blevesearch/bleve/CONTRIBUTING.md | 16 + vendor/github.com/blevesearch/bleve/README.md | 54 +- .../analysis/analyzer/standard/standard.go | 52 + .../custom_analyzer/custom_analyzer.go | 130 - .../detect_lang_analyzer.go | 49 - .../keyword_analyzer/keyword_analyzer.go | 33 - .../simple_analyzer/simple_analyzer.go | 41 - .../standard_analyzer/standard_analyzer.go | 47 - .../ignore/ignore_byte_array_converter.go | 33 - .../json/json_byte_array_converter.go | 40 - .../string/string_byte_array_conveter.go | 33 - .../html_char_filter/html_char_filter.go | 31 - .../regexp_char_filter/regexp_char_filter.go | 58 - .../regexp_char_filter_test.go | 82 - .../zero_width_non_joiner_char_filter.go | 31 - .../analysis/datetime/flexible/flexible.go | 64 + .../analysis/datetime/optional/optional.go | 45 + .../datetime_optional/datetime_optional.go | 40 - .../flexible_go/flexible_go.go | 59 - .../flexible_go/flexible_go_test.go | 84 - .../blevesearch/bleve/analysis/freq.go | 135 +- .../blevesearch/bleve/analysis/freq_test.go | 167 - .../bleve/analysis/lang/en/analyzer_en.go | 70 + .../analysis/lang/en/possessive_filter_en.go | 67 + .../bleve/analysis/lang/en/stop_filter_en.go | 33 + .../{language => lang}/en/stop_words_en.go | 3 +- .../bleve/analysis/language/ar/analyzer_ar.go | 60 - .../analysis/language/ar/analyzer_ar_test.go | 179 - .../analysis/language/ar/arabic_normalize.go | 80 - .../language/ar/arabic_normalize_test.go | 229 - .../bleve/analysis/language/ar/stemmer_ar.go | 113 - .../analysis/language/ar/stemmer_ar_test.go | 392 - .../analysis/language/ar/stop_filter_ar.go | 28 - .../analysis/language/ar/stop_words_ar.go | 149 - .../analysis/language/bg/stop_filter_bg.go | 28 - .../analysis/language/bg/stop_words_bg.go | 217 - .../bleve/analysis/language/ca/articles_ca.go | 30 - .../bleve/analysis/language/ca/elision_ca.go | 32 - .../analysis/language/ca/elision_ca_test.go | 56 - .../analysis/language/ca/stop_filter_ca.go | 28 - .../analysis/language/ca/stop_words_ca.go | 244 - .../analysis/language/cjk/analyzer_cjk.go | 50 - .../language/cjk/analyzer_cjk_test.go | 620 - .../bleve/analysis/language/cjk/cjk_bigram.go | 166 - .../analysis/language/cjk/cjk_bigram_test.go | 420 - .../analysis/language/ckb/analyzer_ckb.go | 58 - .../language/ckb/analyzer_ckb_test.go | 74 - .../analysis/language/ckb/sorani_normalize.go | 113 - .../language/ckb/sorani_normalize_test.go | 318 - .../language/ckb/sorani_stemmer_filter.go | 143 - .../ckb/sorani_stemmer_filter_test.go | 294 - .../analysis/language/ckb/stop_filter_ckb.go | 28 - .../analysis/language/ckb/stop_words_ckb.go | 160 - .../analysis/language/cs/stop_filter_cs.go | 28 - .../analysis/language/cs/stop_words_cs.go | 196 - .../bleve/analysis/language/da/analyzer_da.go | 54 - .../analysis/language/da/analyzer_da_test.go | 69 - .../bleve/analysis/language/da/stemmer_da.go | 28 - .../analysis/language/da/stop_filter_da.go | 28 - .../analysis/language/da/stop_words_da.go | 134 - .../bleve/analysis/language/de/analyzer_de.go | 59 - .../analysis/language/de/analyzer_de_test.go | 97 - .../analysis/language/de/german_normalize.go | 94 - .../language/de/german_normalize_test.go | 98 - .../bleve/analysis/language/de/stemmer_de.go | 28 - .../analysis/language/de/stop_filter_de.go | 28 - .../analysis/language/de/stop_words_de.go | 318 - .../analysis/language/el/stop_filter_el.go | 28 - .../analysis/language/el/stop_words_el.go | 102 - .../bleve/analysis/language/en/analyzer_en.go | 58 - .../analysis/language/en/analyzer_en_test.go | 100 - .../language/en/possessive_filter_en.go | 57 - .../language/en/possessive_filter_en_test.go | 86 - .../bleve/analysis/language/en/stemmer_en.go | 28 - .../analysis/language/en/stemmer_en_test.go | 72 - .../analysis/language/en/stop_filter_en.go | 28 - .../bleve/analysis/language/es/analyzer_es.go | 55 - .../analysis/language/es/analyzer_es_test.go | 64 - .../bleve/analysis/language/es/stemmer_es.go | 28 - .../analysis/language/es/stop_filter_es.go | 28 - .../analysis/language/es/stop_words_es.go | 380 - .../analysis/language/eu/stop_filter_eu.go | 28 - .../analysis/language/eu/stop_words_eu.go | 123 - .../bleve/analysis/language/fa/analyzer_fa.go | 68 - .../analysis/language/fa/analyzer_fa_test.go | 681 - .../analysis/language/fa/persian_normalize.go | 72 - .../language/fa/persian_normalize_test.go | 125 - .../analysis/language/fa/stop_filter_fa.go | 28 - .../analysis/language/fa/stop_words_fa.go | 337 - .../bleve/analysis/language/fi/analyzer_fi.go | 55 - .../analysis/language/fi/analyzer_fi_test.go | 68 - .../bleve/analysis/language/fi/stemmer_fi.go | 28 - .../analysis/language/fi/stop_filter_fi.go | 28 - .../analysis/language/fi/stop_words_fi.go | 121 - .../bleve/analysis/language/fr/analyzer_fr.go | 57 - .../analysis/language/fr/analyzer_fr_test.go | 196 - .../bleve/analysis/language/fr/articles_fr.go | 37 - .../bleve/analysis/language/fr/elision_fr.go | 32 - .../analysis/language/fr/elision_fr_test.go | 50 - .../analysis/language/fr/light_stemmer_fr.go | 308 - .../language/fr/light_stemmer_fr_test.go | 997 -- .../language/fr/minimal_stemmer_fr.go | 81 - .../language/fr/minimal_stemmer_fr_test.go | 134 - .../bleve/analysis/language/fr/stemmer_fr.go | 28 - .../analysis/language/fr/stop_filter_fr.go | 28 - .../analysis/language/fr/stop_words_fr.go | 210 - .../bleve/analysis/language/ga/articles_ga.go | 27 - .../bleve/analysis/language/ga/elision_ga.go | 32 - .../analysis/language/ga/elision_ga_test.go | 50 - .../analysis/language/ga/stop_filter_ga.go | 28 - .../analysis/language/ga/stop_words_ga.go | 134 - .../analysis/language/gl/stop_filter_gl.go | 28 - .../analysis/language/gl/stop_words_gl.go | 185 - .../bleve/analysis/language/hi/analyzer_hi.go | 63 - .../analysis/language/hi/analyzer_hi_test.go | 61 - .../analysis/language/hi/hindi_normalize.go | 133 - .../language/hi/hindi_normalize_test.go | 246 - .../language/hi/hindi_stemmer_filter.go | 144 - .../language/hi/hindi_stemmer_filter_test.go | 303 - .../analysis/language/hi/stop_filter_hi.go | 28 - .../analysis/language/hi/stop_words_hi.go | 259 - .../bleve/analysis/language/hu/analyzer_hu.go | 55 - .../analysis/language/hu/analyzer_hu_test.go | 68 - .../bleve/analysis/language/hu/stemmer_hu.go | 28 - .../analysis/language/hu/stop_filter_hu.go | 28 - .../analysis/language/hu/stop_words_hu.go | 235 - .../analysis/language/hy/stop_filter_hy.go | 28 - .../analysis/language/hy/stop_words_hy.go | 70 - .../analysis/language/id/stop_filter_id.go | 28 - .../analysis/language/id/stop_words_id.go | 383 - .../analysis/language/in/indic_normalize.go | 43 - .../language/in/indic_normalize_test.go | 133 - .../bleve/analysis/language/in/scripts.go | 291 - .../bleve/analysis/language/it/analyzer_it.go | 57 - .../analysis/language/it/analyzer_it_test.go | 82 - .../bleve/analysis/language/it/articles_it.go | 45 - .../bleve/analysis/language/it/elision_it.go | 32 - .../analysis/language/it/elision_it_test.go | 50 - .../analysis/language/it/light_stemmer_it.go | 96 - .../language/it/light_stemmer_it_test.go | 62 - .../bleve/analysis/language/it/stemmer_it.go | 28 - .../analysis/language/it/stop_filter_it.go | 28 - .../analysis/language/it/stop_words_it.go | 327 - .../bleve/analysis/language/ja/analyzer_ja.go | 39 - .../analysis/language/ja/analyzer_ja_test.go | 71 - .../analysis/language/ja/ja_morph_kagome.go | 81 - .../language/ja/ja_morph_kagome_test.go | 56 - .../bleve/analysis/language/nl/analyzer_nl.go | 55 - .../analysis/language/nl/analyzer_nl_test.go | 68 - .../bleve/analysis/language/nl/stemmer_nl.go | 28 - .../analysis/language/nl/stop_filter_nl.go | 28 - .../analysis/language/nl/stop_words_nl.go | 143 - .../bleve/analysis/language/no/analyzer_no.go | 55 - .../analysis/language/no/analyzer_no_test.go | 68 - .../bleve/analysis/language/no/stemmer_no.go | 28 - .../analysis/language/no/stop_filter_no.go | 28 - .../analysis/language/no/stop_words_no.go | 218 - .../language/porter/stemmer_porter.go | 28 - .../bleve/analysis/language/pt/analyzer_pt.go | 52 - .../analysis/language/pt/analyzer_pt_test.go | 65 - .../analysis/language/pt/light_stemmer_pt.go | 190 - .../language/pt/light_stemmer_pt_test.go | 399 - .../bleve/analysis/language/pt/stemmer_pt.go | 28 - .../analysis/language/pt/stop_filter_pt.go | 28 - .../analysis/language/pt/stop_words_pt.go | 277 - .../bleve/analysis/language/ro/analyzer_ro.go | 55 - .../analysis/language/ro/analyzer_ro_test.go | 68 - .../bleve/analysis/language/ro/stemmer_ro.go | 28 - .../analysis/language/ro/stop_filter_ro.go | 28 - .../analysis/language/ro/stop_words_ro.go | 257 - .../bleve/analysis/language/ru/analyzer_ru.go | 55 - .../analysis/language/ru/analyzer_ru_test.go | 98 - .../bleve/analysis/language/ru/stemmer_ru.go | 28 - .../analysis/language/ru/stop_filter_ru.go | 28 - .../analysis/language/ru/stop_words_ru.go | 267 - .../bleve/analysis/language/sv/analyzer_sv.go | 55 - .../analysis/language/sv/analyzer_sv_test.go | 68 - .../bleve/analysis/language/sv/stemmer_sv.go | 28 - .../analysis/language/sv/stop_filter_sv.go | 28 - .../analysis/language/sv/stop_words_sv.go | 157 - .../bleve/analysis/language/th/analyzer_th.go | 48 - .../analysis/language/th/analyzer_th_test.go | 119 - .../analysis/language/th/stop_filter_th.go | 28 - .../analysis/language/th/stop_words_th.go | 143 - .../language/th/unicode_tokenizer_th.go | 28 - .../bleve/analysis/language/tr/analyzer_tr.go | 61 - .../analysis/language/tr/analyzer_tr_test.go | 88 - .../bleve/analysis/language/tr/stemmer_tr.go | 28 - .../analysis/language/tr/stop_filter_tr.go | 28 - .../analysis/language/tr/stop_words_tr.go | 236 - .../lowercase/lowercase.go} | 41 +- .../{token_filters => token}/porter/porter.go | 26 +- .../stop/stop.go} | 36 +- .../apostrophe_filter/apostrophe_filter.go | 49 - .../apostrophe_filter_test.go | 94 - .../analysis/token_filters/cld2/README.md | 33 - .../token_filters/cld2/cld2_filter.cc | 44 - .../token_filters/cld2/cld2_filter.go | 67 - .../analysis/token_filters/cld2/cld2_filter.h | 18 - .../token_filters/cld2/cld2_filter_test.go | 123 - .../token_filters/cld2/compile_cld2.sh | 10 - .../analysis/token_filters/compound/dict.go | 136 - .../token_filters/compound/dict_test.go | 182 - .../edge_ngram_filter/edge_ngram_filter.go | 120 - .../edge_ngram_filter_test.go | 142 - .../elision_filter/elision_filter.go | 67 - .../elision_filter/elision_filter_test.go | 68 - .../keyword_marker_filter.go | 56 - .../keyword_marker_filter_test.go | 68 - .../length_filter/length_filter.go | 72 - .../length_filter/length_filter_test.go | 94 - .../lower_case_filter_test.go | 155 - .../ngram_filter/ngram_filter.go | 90 - .../ngram_filter/ngram_filter_test.go | 132 - .../token_filters/porter/porter_test.go | 58 - .../analysis/token_filters/shingle/shingle.go | 157 - .../token_filters/shingle/shingle_test.go | 330 - .../token_filters/stemmer_filter/README.md | 18 - .../stemmer_filter/stemmer_filter.go | 80 - .../stemmer_filter/stemmer_filter_test.go | 63 - .../stop_tokens_filter_test.go | 73 - .../truncate_token_filter.go | 62 - .../truncate_token_filter_test.go | 74 - .../unicode_normalize/unicode_normalize.go | 74 - .../unicode_normalize_test.go | 157 - .../bleve/analysis/token_map/custom.go | 46 - .../bleve/analysis/token_map_test.go | 38 - .../analysis/tokenizer/unicode/unicode.go | 131 + .../tokenizers/exception/exception.go | 121 - .../tokenizers/exception/exception_test.go | 157 - .../bleve/analysis/tokenizers/icu/boundary.go | 138 - .../analysis/tokenizers/icu/boundary_test.go | 191 - .../regexp_tokenizer/regexp_tokenizer.go | 77 - .../regexp_tokenizer/regexp_tokenizer_test.go | 115 - .../tokenizers/single_token/single_token.go | 44 - .../single_token/single_token_test.go | 71 - .../analysis/tokenizers/unicode/unicode.go | 73 - .../tokenizers/unicode/unicode_test.go | 197 - .../whitespace_tokenizer.go | 30 - .../whitespace_tokenizer_test.go | 150 - .../analysis/{token_map.go => tokenmap.go} | 25 +- .../blevesearch/bleve/analysis/type.go | 38 +- .../blevesearch/bleve/analysis/util.go | 49 +- .../blevesearch/bleve/analysis/util_test.go | 71 - vendor/github.com/blevesearch/bleve/config.go | 171 +- .../blevesearch/bleve/config_app.go | 23 + .../blevesearch/bleve/config_cld2.go | 20 - .../blevesearch/bleve/config_cznicb.go | 16 - .../blevesearch/bleve/config_disk.go | 25 + .../blevesearch/bleve/config_forestdb.go | 16 - .../blevesearch/bleve/config_icu.go | 16 - .../blevesearch/bleve/config_kagome.go | 16 - .../blevesearch/bleve/config_leveldb.go | 21 - .../blevesearch/bleve/config_metrics.go | 17 - .../blevesearch/bleve/config_rocksdb.go | 16 - .../blevesearch/bleve/config_stemmer.go | 18 - vendor/github.com/blevesearch/bleve/doc.go | 19 +- .../blevesearch/bleve/docs/bleve.png | Bin 6727 -> 0 bytes .../blevesearch/bleve/docs/build_children.sh | 11 - .../bleve/docs/merge-coverprofile.go | 45 - .../bleve/docs/old_build_script.txt | 29 - .../bleve/docs/project-code-coverage.sh | 51 - .../blevesearch/bleve/document/document.go | 39 +- .../blevesearch/bleve/document/field.go | 31 +- .../bleve/document/field_boolean.go | 107 + .../bleve/document/field_composite.go | 43 +- .../bleve/document/field_datetime.go | 52 +- .../bleve/document/field_geopoint.go | 137 + .../bleve/document/field_numeric.go | 56 +- .../bleve/document/field_numeric_test.go | 25 - .../blevesearch/bleve/document/field_text.go | 69 +- .../bleve/document/indexing_options.go | 19 +- .../bleve/document/indexing_options_test.go | 69 - vendor/github.com/blevesearch/bleve/error.go | 55 +- .../blevesearch/bleve/examples_test.go | 485 - .../blevesearch/bleve/geo/README.md | 9 + .../github.com/blevesearch/bleve/geo/geo.go | 205 + .../blevesearch/bleve/geo/geo_dist.go | 98 + .../github.com/blevesearch/bleve/geo/parse.go | 140 + .../blevesearch/bleve/geo/sloppy.go | 206 + .../blevesearch/bleve/http/alias.go | 65 - .../blevesearch/bleve/http/debug.go | 74 - .../blevesearch/bleve/http/doc_count.go | 56 - .../blevesearch/bleve/http/doc_delete.go | 67 - .../blevesearch/bleve/http/doc_get.go | 107 - .../blevesearch/bleve/http/doc_index.go | 75 - .../blevesearch/bleve/http/fields.go | 58 - .../blevesearch/bleve/http/handlers_test.go | 704 - .../blevesearch/bleve/http/index_create.go | 78 - .../blevesearch/bleve/http/index_delete.go | 70 - .../blevesearch/bleve/http/index_get.go | 54 - .../blevesearch/bleve/http/index_list.go | 33 - .../blevesearch/bleve/http/registry.go | 120 - .../blevesearch/bleve/http/search.go | 84 - .../github.com/blevesearch/bleve/http/util.go | 46 - vendor/github.com/blevesearch/bleve/index.go | 153 +- .../blevesearch/bleve/index/analysis.go | 83 + .../blevesearch/bleve/index/field_cache.go | 88 + .../blevesearch/bleve/index/index.go | 147 +- .../blevesearch/bleve/index/store/batch.go | 71 +- .../bleve/index/store/boltdb/iterator.go | 49 +- .../bleve/index/store/boltdb/reader.go | 62 +- .../bleve/index/store/boltdb/stats.go | 26 + .../bleve/index/store/boltdb/store.go | 195 +- .../bleve/index/store/boltdb/store_test.go | 272 - .../bleve/index/store/boltdb/writer.go | 113 +- .../bleve/index/store/cznicb/batch.go | 88 - .../bleve/index/store/cznicb/cznicb.go | 112 - .../bleve/index/store/cznicb/cznicb_test.go | 133 - .../bleve/index/store/cznicb/iterator.go | 113 - .../bleve/index/store/cznicb/reader.go | 44 - .../bleve/index/store/cznicb/writer.go | 55 - .../bleve/index/store/forestdb/batch.go | 86 - .../bleve/index/store/forestdb/iterator.go | 117 - .../bleve/index/store/forestdb/reader.go | 57 - .../bleve/index/store/forestdb/store.go | 297 - .../bleve/index/store/forestdb/store_test.go | 636 - .../bleve/index/store/forestdb/writer.go | 71 - .../bleve/index/store/goleveldb/batch.go | 53 - .../bleve/index/store/goleveldb/iterator.go | 75 - .../bleve/index/store/goleveldb/reader.go | 47 - .../bleve/index/store/goleveldb/store.go | 172 - .../bleve/index/store/goleveldb/store_test.go | 296 - .../bleve/index/store/goleveldb/util.go | 26 - .../bleve/index/store/goleveldb/writer.go | 63 - .../bleve/index/store/gorocksdb/batch.go | 43 - .../bleve/index/store/gorocksdb/iterator.go | 76 - .../bleve/index/store/gorocksdb/reader.go | 48 - .../bleve/index/store/gorocksdb/store.go | 146 - .../bleve/index/store/gorocksdb/store_test.go | 298 - .../bleve/index/store/gorocksdb/util.go | 28 - .../bleve/index/store/gorocksdb/writer.go | 64 - .../bleve/index/store/gtreap/gtreap.go | 96 - .../bleve/index/store/gtreap/gtreap_test.go | 259 - .../bleve/index/store/gtreap/iterator.go | 62 +- .../bleve/index/store/gtreap/reader.go | 53 +- .../bleve/index/store/gtreap/store.go | 82 + .../bleve/index/store/gtreap/writer.go | 114 +- .../bleve/index/store/inmem/iterator.go | 70 - .../bleve/index/store/inmem/reader.go | 40 - .../bleve/index/store/inmem/store.go | 106 - .../bleve/index/store/inmem/store_test.go | 254 - .../bleve/index/store/inmem/writer.go | 57 - .../blevesearch/bleve/index/store/kvstore.go | 185 +- .../bleve/index/store/leveldb/batch.go | 55 - .../bleve/index/store/leveldb/iterator.go | 78 - .../bleve/index/store/leveldb/reader.go | 48 - .../bleve/index/store/leveldb/store.go | 174 - .../bleve/index/store/leveldb/store_test.go | 298 - .../bleve/index/store/leveldb/util.go | 28 - .../bleve/index/store/leveldb/writer.go | 65 - .../blevesearch/bleve/index/store/merge.go | 88 +- .../bleve/index/store/metrics/metrics.go | 476 - .../bleve/index/store/metrics/metrics_test.go | 368 - .../blevesearch/bleve/index/store/multiget.go | 33 + .../bleve/index/store/null/null.go | 173 - .../bleve/index/store/null/null_test.go | 88 - .../bleve/index/upside_down/analysis_pool.go | 121 - .../upside_down/benchmark_boltdb_test.go | 77 - .../upside_down/benchmark_common_test.go | 143 - .../upside_down/benchmark_cznicb_test.go | 75 - .../upside_down/benchmark_forestdb_test.go | 86 - .../upside_down/benchmark_goleveldb_test.go | 80 - .../upside_down/benchmark_gorocksdb_test.go | 82 - .../upside_down/benchmark_gtreap_test.go | 75 - .../index/upside_down/benchmark_inmem_test.go | 75 - .../upside_down/benchmark_leveldb_test.go | 82 - .../index/upside_down/benchmark_null_test.go | 75 - .../bleve/index/upside_down/dump.go | 177 - .../bleve/index/upside_down/dump_test.go | 127 - .../bleve/index/upside_down/field_dict.go | 86 - .../index/upside_down/field_dict_test.go | 180 - .../index/upside_down/field_index_cache.go | 69 - .../bleve/index/upside_down/index_reader.go | 164 - .../bleve/index/upside_down/reader.go | 187 - .../bleve/index/upside_down/reader_test.go | 297 - .../bleve/index/upside_down/row.go | 641 - .../bleve/index/upside_down/row_test.go | 329 - .../bleve/index/upside_down/stats.go | 31 - .../bleve/index/upside_down/upside_down.go | 721 - .../bleve/index/upside_down/upside_down.pb.go | 98 - .../index/upside_down/upside_down_test.go | 1177 -- .../bleve/index/upsidedown/analysis.go | 110 + .../benchmark_all.sh | 0 .../bleve/index/upsidedown/dump.go | 174 + .../bleve/index/upsidedown/field_dict.go | 78 + .../bleve/index/upsidedown/index_reader.go | 203 + .../bleve/index/upsidedown/reader.go | 336 + .../blevesearch/bleve/index/upsidedown/row.go | 1112 ++ .../{upside_down => upsidedown}/row_merge.go | 26 +- .../bleve/index/upsidedown/stats.go | 55 + .../bleve/index/upsidedown/upsidedown.go | 1072 ++ .../bleve/index/upsidedown/upsidedown.pb.go | 688 + .../upsidedown.proto} | 10 +- .../blevesearch/bleve/index_alias.go | 19 +- .../blevesearch/bleve/index_alias_impl.go | 252 +- .../bleve/index_alias_impl_test.go | 790 - .../blevesearch/bleve/index_impl.go | 431 +- .../blevesearch/bleve/index_meta.go | 43 +- .../blevesearch/bleve/index_meta_test.go | 54 - .../blevesearch/bleve/index_stats.go | 66 +- .../blevesearch/bleve/index_test.go | 609 - .../github.com/blevesearch/bleve/mapping.go | 65 + .../blevesearch/bleve/mapping/analysis.go | 99 + .../document.go} | 312 +- .../blevesearch/bleve/mapping/field.go | 321 + .../blevesearch/bleve/mapping/index.go | 430 + .../blevesearch/bleve/mapping/mapping.go | 49 + .../bleve/{ => mapping}/reflect.go | 34 +- .../blevesearch/bleve/mapping_field.go | 155 - .../blevesearch/bleve/mapping_index.go | 455 - .../blevesearch/bleve/mapping_test.go | 246 - .../blevesearch/bleve/numeric/bin.go | 43 + .../blevesearch/bleve/numeric/float.go | 34 + .../{numeric_util => numeric}/prefix_coded.go | 42 +- .../blevesearch/bleve/numeric_util/float.go | 29 - .../bleve/numeric_util/float_test.go | 59 - .../bleve/numeric_util/prefix_coded_test.go | 98 - vendor/github.com/blevesearch/bleve/query.go | 407 +- .../blevesearch/bleve/query_boolean.go | 210 - .../blevesearch/bleve/query_conjunction.go | 94 - .../blevesearch/bleve/query_date_range.go | 117 - .../blevesearch/bleve/query_disjunction.go | 119 - .../blevesearch/bleve/query_fuzzy.go | 88 - .../blevesearch/bleve/query_match.go | 128 - .../blevesearch/bleve/query_match_all.go | 53 - .../blevesearch/bleve/query_match_none.go | 53 - .../blevesearch/bleve/query_match_phrase.go | 114 - .../blevesearch/bleve/query_numeric_range.go | 81 - .../blevesearch/bleve/query_phrase.go | 108 - .../blevesearch/bleve/query_prefix.go | 62 - .../blevesearch/bleve/query_regexp.go | 75 - .../blevesearch/bleve/query_string.go | 59 - .../blevesearch/bleve/query_string.nex | 41 - .../blevesearch/bleve/query_string.nn.go | 1362 -- .../blevesearch/bleve/query_string.y | 217 - .../blevesearch/bleve/query_string.y.go | 563 - .../blevesearch/bleve/query_string_parser.go | 76 - .../bleve/query_string_parser_test.go | 247 - .../blevesearch/bleve/query_term.go | 61 - .../blevesearch/bleve/query_test.go | 220 - .../blevesearch/bleve/query_wildcard.go | 102 - .../blevesearch/bleve/registry/analyzer.go | 71 +- .../bleve/registry/byte_array_converter.go | 47 - .../blevesearch/bleve/registry/cache.go | 87 + .../blevesearch/bleve/registry/char_filter.go | 71 +- .../bleve/registry/datetime_parser.go | 79 +- .../bleve/registry/fragment_formatter.go | 71 +- .../blevesearch/bleve/registry/fragmenter.go | 71 +- .../blevesearch/bleve/registry/highlighter.go | 71 +- .../blevesearch/bleve/registry/index_type.go | 45 + .../blevesearch/bleve/registry/registry.go | 72 +- .../blevesearch/bleve/registry/store.go | 41 +- .../bleve/registry/token_filter.go | 71 +- .../blevesearch/bleve/registry/token_maps.go | 71 +- .../blevesearch/bleve/registry/tokenizer.go | 71 +- vendor/github.com/blevesearch/bleve/search.go | 240 +- .../blevesearch/bleve/search/collector.go | 25 +- .../bleve/search/collector/heap.go | 91 + .../bleve/search/collector/list.go | 78 + .../bleve/search/collector/slice.go | 68 + .../bleve/search/collector/topn.go | 278 + .../search/collectors/collector_top_score.go | 135 - .../collectors/collector_top_score_test.go | 249 - .../bleve/search/collectors/search_test.go | 60 - .../blevesearch/bleve/search/explanation.go | 19 +- .../{facets => facet}/benchmark_data.txt | 0 .../search/facet/facet_builder_datetime.go | 134 + .../facet_builder_numeric.go | 76 +- .../{facets => facet}/facet_builder_terms.go | 54 +- .../search/facets/facet_builder_datetime.go | 147 - .../facets/facet_builder_numeric_test.go | 49 - .../search/facets/facet_builder_terms_test.go | 58 - .../bleve/search/facets_builder.go | 135 +- .../bleve/search/facets_builder_test.go | 301 - .../html/html.go} | 27 +- .../ansi/fragment_formatter_ansi.go | 100 - .../html/fragment_formatter_html_test.go | 87 - .../simple/simple.go} | 28 +- .../simple/fragmenter_simple_test.go | 295 - .../bleve/search/highlight/highlighter.go | 30 +- .../search/highlight/highlighter/html/html.go | 50 + .../simple/fragment_scorer_simple.go | 21 +- .../simple/highlighter_simple.go | 35 +- .../simple/fragment_scorer_simple_test.go | 77 - .../simple/highlighter_simple_test.go | 164 - .../bleve/search/highlight/term_locations.go | 79 +- .../search/highlight/term_locations_test.go | 173 - .../blevesearch/bleve/search/levenshtein.go | 34 +- .../bleve/search/levenshtein_test.go | 114 - .../blevesearch/bleve/search/pool.go | 80 + .../bleve/search/query/bool_field.go | 64 + .../blevesearch/bleve/search/query/boolean.go | 248 + .../blevesearch/bleve/search/query/boost.go | 33 + .../bleve/search/query/conjunction.go | 110 + .../bleve/search/query/date_range.go | 164 + .../bleve/search/query/disjunction.go | 121 + .../blevesearch/bleve/search/query/docid.go | 49 + .../blevesearch/bleve/search/query/fuzzy.go | 77 + .../bleve/search/query/geo_boundingbox.go | 113 + .../bleve/search/query/geo_distance.go | 100 + .../blevesearch/bleve/search/query/match.go | 176 + .../bleve/search/query/match_all.go | 55 + .../bleve/search/query/match_none.go | 55 + .../bleve/search/query/match_phrase.go | 113 + .../bleve/search/query/multi_phrase.go | 80 + .../bleve/search/query/numeric_range.go | 87 + .../blevesearch/bleve/search/query/phrase.go | 77 + .../blevesearch/bleve/search/query/prefix.go | 62 + .../blevesearch/bleve/search/query/query.go | 356 + .../bleve/search/query/query_string.go | 63 + .../bleve/search/query/query_string.y | 328 + .../bleve/search/query/query_string.y.go | 815 ++ .../bleve/search/query/query_string_lex.go | 322 + .../bleve/search/query/query_string_parser.go | 85 + .../blevesearch/bleve/search/query/regexp.go | 96 + .../blevesearch/bleve/search/query/term.go | 61 + .../bleve/search/query/term_range.go | 95 + .../bleve/search/query/wildcard.go | 106 + .../bleve/search/scorer/scorer_conjunction.go | 65 + .../{scorers => scorer}/scorer_constant.go | 47 +- .../bleve/search/scorer/scorer_disjunction.go | 77 + .../search/{scorers => scorer}/scorer_term.go | 89 +- .../bleve/search/scorer/sqrt_cache.go | 30 + .../search/scorers/scorer_conjunction.go | 59 - .../search/scorers/scorer_constant_test.go | 118 - .../search/scorers/scorer_disjunction.go | 71 - .../bleve/search/scorers/scorer_term_test.go | 241 - .../bleve/search/scorers/sqrt_cache.go | 25 - .../blevesearch/bleve/search/search.go | 140 +- .../search/searcher/ordered_searchers_list.go | 35 + .../bleve/search/searcher/search_boolean.go | 391 + .../search/searcher/search_conjunction.go | 232 + .../search/searcher/search_disjunction.go | 271 + .../bleve/search/searcher/search_docid.go | 93 + .../bleve/search/searcher/search_filter.go | 88 + .../bleve/search/searcher/search_fuzzy.go | 74 + .../search/searcher/search_geoboundingbox.go | 173 + .../searcher/search_geopointdistance.go | 115 + .../bleve/search/searcher/search_match_all.go | 105 + .../search/searcher/search_match_none.go | 62 + .../search/searcher/search_multi_term.go | 85 + .../search_numeric_range.go | 105 +- .../bleve/search/searcher/search_phrase.go | 341 + .../bleve/search/searcher/search_regexp.go | 80 + .../bleve/search/searcher/search_term.go | 122 + .../search/searcher/search_term_prefix.go | 39 + .../search/searcher/search_term_range.go | 75 + .../bleve/search/searchers/base_test.go | 80 - .../searchers/ordered_searchers_list.go | 30 - .../bleve/search/searchers/search_boolean.go | 335 - .../search/searchers/search_boolean_test.go | 364 - .../search/searchers/search_conjunction.go | 197 - .../searchers/search_conjunction_test.go | 212 - .../search/searchers/search_disjunction.go | 189 - .../searchers/search_disjunction_test.go | 168 - .../bleve/search/searchers/search_fuzzy.go | 112 - .../search/searchers/search_match_all.go | 89 - .../search/searchers/search_match_all_test.go | 134 - .../search/searchers/search_match_none.go | 53 - .../searchers/search_match_none_test.go | 76 - .../searchers/search_numeric_range_test.go | 55 - .../bleve/search/searchers/search_phrase.go | 197 - .../search/searchers/search_phrase_test.go | 93 - .../bleve/search/searchers/search_regexp.go | 108 - .../search/searchers/search_regexp_test.go | 110 - .../bleve/search/searchers/search_term.go | 95 - .../search/searchers/search_term_prefix.go | 81 - .../search/searchers/search_term_test.go | 195 - .../blevesearch/bleve/search/sort.go | 711 + .../blevesearch/bleve/search/util.go | 19 +- .../blevesearch/bleve/search/util_test.go | 86 - .../blevesearch/bleve/search_test.go | 117 - .../blevesearch/bleve/test/integration.go | 22 - .../bleve/test/integration_test.go | 163 - .../bleve/test/tests/basic/data/a.json | 7 - .../bleve/test/tests/basic/data/b.json | 7 - .../bleve/test/tests/basic/data/c.json | 7 - .../bleve/test/tests/basic/data/d.json | 7 - .../bleve/test/tests/basic/mapping.json | 27 - .../bleve/test/tests/basic/searches.json | 406 - .../bleve/test/tests/facet/data/a.json | 6 - .../bleve/test/tests/facet/data/b.json | 6 - .../bleve/test/tests/facet/data/c.json | 6 - .../bleve/test/tests/facet/data/d.json | 6 - .../bleve/test/tests/facet/data/e.json | 6 - .../bleve/test/tests/facet/data/f.json | 6 - .../bleve/test/tests/facet/data/g.json | 6 - .../bleve/test/tests/facet/data/h.json | 6 - .../bleve/test/tests/facet/data/i.json | 6 - .../bleve/test/tests/facet/data/j.json | 6 - .../bleve/test/tests/facet/mapping.json | 1 - .../bleve/test/tests/facet/searches.json | 144 - .../fosdem/data/3311@FOSDEM15@fosdem.org.json | 4 - .../fosdem/data/3492@FOSDEM15@fosdem.org.json | 4 - .../fosdem/data/3496@FOSDEM15@fosdem.org.json | 4 - .../fosdem/data/3505@FOSDEM15@fosdem.org.json | 4 - .../fosdem/data/3507@FOSDEM15@fosdem.org.json | 4 - .../bleve/test/tests/fosdem/mapping.json | 76 - .../bleve/test/tests/fosdem/searches.json | 105 - .../bleve/test/tests/phrase/data/a.json | 3 - .../bleve/test/tests/phrase/mapping.json | 23 - .../bleve/test/tests/phrase/searches.json | 326 - .../bleve/utils/bleve_bulkindex/main.go | 104 - .../bleve/utils/bleve_create/main.go | 58 - .../bleve/utils/bleve_dump/main.go | 73 - .../bleve/utils/bleve_index/main.go | 115 - .../bleve/utils/bleve_query/main.go | 85 - .../bleve/utils/bleve_registry/main.go | 82 - .../porterstemmer_contains_vowel_test.go | 57 - .../porterstemmer_fixes_test.go | 20 - .../porterstemmer_fuzz_test.go | 59 - ...has_repeat_double_consonant_suffix_test.go | 42 - .../porterstemmer_has_suffix_test.go | 432 - .../porterstemmer_is_consontant_test.go | 74 - .../porterstemmer_measure_test.go | 99 - .../porterstemmer_stem_string_test.go | 118 - ...rstemmer_stem_without_lower_casing_test.go | 56 - .../porterstemmer_step1a_test.go | 70 - .../porterstemmer_step1b_test.go | 112 - .../porterstemmer_step1c_test.go | 60 - .../porterstemmer_step2_test.go | 132 - .../porterstemmer_step3_test.go | 76 - .../porterstemmer_step4_test.go | 124 - .../porterstemmer_step5a_test.go | 60 - .../porterstemmer_step5b_test.go | 56 - .../blevesearch/segment/export_test.go | 20 - .../segment/ragel/unicode2ragel.rb | 330 - .../blevesearch/segment/ragel/uscript.rl | 101 - .../blevesearch/segment/ragel/uwb.rl | 1290 -- .../blevesearch/segment/segment_fuzz_test.go | 29 - .../blevesearch/segment/segment_test.go | 241 - .../blevesearch/segment/segment_words_test.go | 445 - .../blevesearch/segment/tables_test.go | 11994 ---------------- vendor/github.com/boltdb/bolt/README.md | 98 +- vendor/github.com/boltdb/bolt/bolt_386.go | 3 + vendor/github.com/boltdb/bolt/bolt_amd64.go | 3 + vendor/github.com/boltdb/bolt/bolt_arm.go | 21 + vendor/github.com/boltdb/bolt/bolt_arm64.go | 3 + vendor/github.com/boltdb/bolt/bolt_ppc.go | 9 + vendor/github.com/boltdb/bolt/bolt_ppc64.go | 12 + vendor/github.com/boltdb/bolt/bolt_ppc64le.go | 3 + vendor/github.com/boltdb/bolt/bolt_s390x.go | 3 + vendor/github.com/boltdb/bolt/bolt_unix.go | 8 +- .../boltdb/bolt/bolt_unix_solaris.go | 8 +- vendor/github.com/boltdb/bolt/bolt_windows.go | 22 +- vendor/github.com/boltdb/bolt/bucket.go | 35 +- vendor/github.com/boltdb/bolt/bucket_test.go | 1827 --- .../github.com/boltdb/bolt/cmd/bolt/main.go | 1532 -- .../boltdb/bolt/cmd/bolt/main_test.go | 185 - vendor/github.com/boltdb/bolt/cursor_test.go | 817 -- vendor/github.com/boltdb/bolt/db.go | 99 +- vendor/github.com/boltdb/bolt/db_test.go | 1684 --- vendor/github.com/boltdb/bolt/errors.go | 3 +- vendor/github.com/boltdb/bolt/freelist.go | 48 +- .../github.com/boltdb/bolt/freelist_test.go | 158 - vendor/github.com/boltdb/bolt/node.go | 42 +- vendor/github.com/boltdb/bolt/node_test.go | 156 - vendor/github.com/boltdb/bolt/page.go | 41 +- vendor/github.com/boltdb/bolt/page_test.go | 72 - vendor/github.com/boltdb/bolt/quick_test.go | 79 - .../github.com/boltdb/bolt/simulation_test.go | 329 - vendor/github.com/boltdb/bolt/tx.go | 70 +- vendor/github.com/boltdb/bolt/tx_test.go | 716 - .../cpuguy83/go-md2man/md2man/md2man.go | 19 - .../cpuguy83/go-md2man/md2man/roff.go | 281 - vendor/github.com/cznic/b/AUTHORS | 11 - vendor/github.com/cznic/b/CONTRIBUTORS | 12 - vendor/github.com/cznic/b/Makefile | 53 - vendor/github.com/cznic/b/README.md | 10 - vendor/github.com/cznic/b/all_test.go | 1349 -- vendor/github.com/cznic/b/btree.go | 929 -- vendor/github.com/cznic/b/doc.go | 53 - vendor/github.com/cznic/b/example/Makefile | 35 - vendor/github.com/cznic/b/example/all_test.go | 1126 -- vendor/github.com/cznic/b/example/int.go | 929 -- vendor/github.com/davecgh/go-spew/LICENSE | 15 + .../github.com/davecgh/go-spew/spew/bypass.go | 9 +- .../davecgh/go-spew/spew/bypasssafe.go | 9 +- .../github.com/davecgh/go-spew/spew/common.go | 2 +- .../davecgh/go-spew/spew/common_test.go | 298 - .../github.com/davecgh/go-spew/spew/config.go | 13 +- vendor/github.com/davecgh/go-spew/spew/doc.go | 11 +- .../github.com/davecgh/go-spew/spew/dump.go | 8 +- .../davecgh/go-spew/spew/dump_test.go | 1042 -- .../davecgh/go-spew/spew/dumpcgo_test.go | 98 - .../davecgh/go-spew/spew/dumpnocgo_test.go | 26 - .../davecgh/go-spew/spew/example_test.go | 226 - .../github.com/davecgh/go-spew/spew/format.go | 2 +- .../davecgh/go-spew/spew/format_test.go | 1558 -- .../davecgh/go-spew/spew/internal_test.go | 87 - .../go-spew/spew/internalunsafe_test.go | 101 - .../github.com/davecgh/go-spew/spew/spew.go | 2 +- .../davecgh/go-spew/spew/spew_test.go | 309 - .../davecgh/go-spew/spew/testdata/dumpcgo.go | 82 - .../dgrijalva/jwt-go/MIGRATION_GUIDE.md | 97 + vendor/github.com/dgrijalva/jwt-go/README.md | 49 +- .../dgrijalva/jwt-go/VERSION_HISTORY.md | 38 + vendor/github.com/dgrijalva/jwt-go/claims.go | 134 + .../dgrijalva/jwt-go/cmd/jwt/app.go | 210 - vendor/github.com/dgrijalva/jwt-go/ecdsa.go | 4 +- .../github.com/dgrijalva/jwt-go/ecdsa_test.go | 100 - vendor/github.com/dgrijalva/jwt-go/errors.go | 40 +- .../dgrijalva/jwt-go/example_test.go | 52 - vendor/github.com/dgrijalva/jwt-go/hmac.go | 2 +- .../github.com/dgrijalva/jwt-go/hmac_test.go | 91 - .../github.com/dgrijalva/jwt-go/map_claims.go | 94 + vendor/github.com/dgrijalva/jwt-go/none.go | 52 + vendor/github.com/dgrijalva/jwt-go/parser.go | 69 +- .../dgrijalva/jwt-go/parser_test.go | 239 - vendor/github.com/dgrijalva/jwt-go/rsa.go | 30 +- vendor/github.com/dgrijalva/jwt-go/rsa_pss.go | 2 +- .../dgrijalva/jwt-go/rsa_pss_test.go | 96 - .../github.com/dgrijalva/jwt-go/rsa_test.go | 174 - .../github.com/dgrijalva/jwt-go/rsa_utils.go | 3 +- .../dgrijalva/jwt-go/signing_method.go | 11 + .../dgrijalva/jwt-go/test/ec256-private.pem | 5 - .../dgrijalva/jwt-go/test/ec256-public.pem | 4 - .../dgrijalva/jwt-go/test/ec384-private.pem | 6 - .../dgrijalva/jwt-go/test/ec384-public.pem | 5 - .../dgrijalva/jwt-go/test/ec512-private.pem | 7 - .../dgrijalva/jwt-go/test/ec512-public.pem | 6 - .../dgrijalva/jwt-go/test/hmacTestKey | 1 - .../dgrijalva/jwt-go/test/sample_key | 27 - .../dgrijalva/jwt-go/test/sample_key.pub | 9 - vendor/github.com/dgrijalva/jwt-go/token.go | 52 +- vendor/github.com/dsnet/compress/LICENSE.md | 24 + .../dsnet/compress/brotli/brotli_test.go | 5 - .../dsnet/compress/brotli/common_test.go | 225 - .../compress/brotli/dict_decoder_test.go | 161 - .../compress/brotli/dict_encoder_test.go | 5 - .../dsnet/compress/brotli/prefix_test.go | 5 - .../dsnet/compress/brotli/reader_test.go | 455 - .../dsnet/compress/brotli/transform_test.go | 61 - .../dsnet/compress/brotli/writer_test.go | 5 - .../fsnotify/fsnotify}/AUTHORS | 6 + .../fsnotify/fsnotify}/CHANGELOG.md | 74 +- .../fsnotify/fsnotify}/CONTRIBUTING.md | 16 +- .../fsnotify/fsnotify}/LICENSE | 0 vendor/github.com/fsnotify/fsnotify/README.md | 79 + vendor/github.com/fsnotify/fsnotify/fen.go | 37 + .../fsnotify/fsnotify}/fsnotify.go | 34 +- .../fsnotify/fsnotify}/inotify.go | 123 +- .../fsnotify/fsnotify}/inotify_poller.go | 53 +- .../fsnotify/fsnotify}/kqueue.go | 51 +- .../fsnotify/fsnotify}/open_mode_bsd.go | 4 +- .../fsnotify/fsnotify}/open_mode_darwin.go | 4 +- .../fsnotify/fsnotify}/windows.go | 96 +- vendor/github.com/golang/protobuf/LICENSE | 31 + .../github.com/golang/protobuf/proto/Makefile | 2 +- .../golang/protobuf/proto/all_test.go | 2169 --- .../github.com/golang/protobuf/proto/clone.go | 12 +- .../golang/protobuf/proto/clone_test.go | 267 - .../golang/protobuf/proto/decode.go | 133 +- .../golang/protobuf/proto/encode.go | 81 +- .../github.com/golang/protobuf/proto/equal.go | 34 +- .../golang/protobuf/proto/equal_test.go | 212 - .../golang/protobuf/proto/extensions.go | 228 +- .../golang/protobuf/proto/extensions_test.go | 430 - .../github.com/golang/protobuf/proto/lib.go | 6 +- .../golang/protobuf/proto/message_set.go | 43 +- .../golang/protobuf/proto/message_set_test.go | 66 - .../golang/protobuf/proto/pointer_reflect.go | 7 +- .../golang/protobuf/proto/pointer_unsafe.go | 6 +- .../golang/protobuf/proto/properties.go | 56 +- .../protobuf/proto/proto3_proto/proto3.pb.go | 122 - .../protobuf/proto/proto3_proto/proto3.proto | 68 - .../golang/protobuf/proto/proto3_test.go | 125 - .../golang/protobuf/proto/size2_test.go | 63 - .../golang/protobuf/proto/size_test.go | 164 - .../golang/protobuf/proto/testdata/Makefile | 50 - .../protobuf/proto/testdata/golden_test.go | 86 - .../golang/protobuf/proto/testdata/test.pb.go | 3992 ----- .../golang/protobuf/proto/testdata/test.proto | 535 - .../github.com/golang/protobuf/proto/text.go | 148 +- .../golang/protobuf/proto/text_parser.go | 189 +- .../golang/protobuf/proto/text_parser_test.go | 557 - .../golang/protobuf/proto/text_test.go | 474 - vendor/github.com/golang/snappy/AUTHORS | 15 - vendor/github.com/golang/snappy/CONTRIBUTORS | 37 - vendor/github.com/golang/snappy/README | 7 - .../golang/snappy/cmd/snappytool/main.cpp | 74 - vendor/github.com/golang/snappy/decode.go | 237 - .../github.com/golang/snappy/decode_amd64.go | 10 - .../github.com/golang/snappy/decode_amd64.s | 472 - .../github.com/golang/snappy/decode_other.go | 96 - vendor/github.com/golang/snappy/encode.go | 403 - vendor/github.com/golang/snappy/snappy.go | 84 - .../github.com/golang/snappy/snappy_test.go | 973 -- .../github.com/golang/snappy/testdata/pi.txt | 1 - .../golang/snappy/testdata/pi.txt.rawsnappy | Bin 8929 -> 0 bytes vendor/github.com/gorilla/websocket/README.md | 5 +- .../gorilla/websocket/bench_test.go | 19 - vendor/github.com/gorilla/websocket/client.go | 60 +- .../gorilla/websocket/client_clone.go | 16 + .../gorilla/websocket/client_clone_legacy.go | 38 + .../gorilla/websocket/client_server_test.go | 451 - .../gorilla/websocket/client_test.go | 72 - .../gorilla/websocket/compression.go | 148 + vendor/github.com/gorilla/websocket/conn.go | 632 +- .../github.com/gorilla/websocket/conn_read.go | 18 + .../gorilla/websocket/conn_read_legacy.go | 21 + .../github.com/gorilla/websocket/conn_test.go | 367 - vendor/github.com/gorilla/websocket/doc.go | 54 +- .../gorilla/websocket/example_test.go | 40 - .../websocket/examples/autobahn/README.md | 13 - .../examples/autobahn/fuzzingclient.json | 14 - .../websocket/examples/autobahn/server.go | 246 - .../gorilla/websocket/examples/chat/README.md | 20 - .../gorilla/websocket/examples/chat/conn.go | 105 - .../gorilla/websocket/examples/chat/home.html | 92 - .../gorilla/websocket/examples/chat/hub.go | 51 - .../gorilla/websocket/examples/chat/main.go | 39 - .../websocket/examples/command/README.md | 19 - .../websocket/examples/command/home.html | 96 - .../websocket/examples/command/main.go | 188 - .../gorilla/websocket/examples/echo/README.md | 17 - .../gorilla/websocket/examples/echo/client.go | 81 - .../gorilla/websocket/examples/echo/server.go | 132 - .../websocket/examples/filewatch/README.md | 9 - .../websocket/examples/filewatch/main.go | 193 - .../github.com/gorilla/websocket/json_test.go | 119 - vendor/github.com/gorilla/websocket/mask.go | 55 + .../github.com/gorilla/websocket/mask_safe.go | 15 + .../github.com/gorilla/websocket/prepared.go | 103 + vendor/github.com/gorilla/websocket/server.go | 68 +- .../gorilla/websocket/server_test.go | 33 - vendor/github.com/gorilla/websocket/util.go | 196 +- .../github.com/gorilla/websocket/util_test.go | 34 - vendor/github.com/hashicorp/hcl/Makefile | 1 + vendor/github.com/hashicorp/hcl/README.md | 29 +- vendor/github.com/hashicorp/hcl/appveyor.yml | 19 + vendor/github.com/hashicorp/hcl/decoder.go | 107 +- .../github.com/hashicorp/hcl/decoder_test.go | 669 - .../github.com/hashicorp/hcl/hcl/ast/ast.go | 12 +- .../hashicorp/hcl/hcl/ast/ast_test.go | 200 - .../hashicorp/hcl/hcl/fmtcmd/fmtcmd.go | 164 - .../hashicorp/hcl/hcl/fmtcmd/fmtcmd_test.go | 431 - .../hcl/hcl/fmtcmd/test-fixtures/dir.ignore | 0 .../hcl/hcl/fmtcmd/test-fixtures/file.ignore | 1 - .../hcl/hcl/fmtcmd/test-fixtures/good.hcl | 0 .../hashicorp/hcl/hcl/parser/error_test.go | 9 - .../hashicorp/hcl/hcl/parser/parser.go | 134 +- .../hashicorp/hcl/hcl/parser/parser_test.go | 330 - .../parser/test-fixtures/array_comment.hcl | 4 - .../parser/test-fixtures/array_comment_2.hcl | 6 - .../hcl/parser/test-fixtures/assign_colon.hcl | 6 - .../hcl/parser/test-fixtures/assign_deep.hcl | 5 - .../hcl/hcl/parser/test-fixtures/comment.hcl | 15 - .../parser/test-fixtures/comment_lastline.hcl | 1 - .../parser/test-fixtures/comment_single.hcl | 1 - .../hcl/hcl/parser/test-fixtures/complex.hcl | 42 - .../hcl/parser/test-fixtures/complex_key.hcl | 1 - .../hcl/hcl/parser/test-fixtures/empty.hcl | 0 .../hcl/hcl/parser/test-fixtures/list.hcl | 1 - .../hcl/parser/test-fixtures/list_comma.hcl | 1 - .../parser/test-fixtures/missing_braces.hcl | 4 - .../hcl/hcl/parser/test-fixtures/multiple.hcl | 2 - .../hcl/hcl/parser/test-fixtures/old.hcl | 3 - .../hcl/parser/test-fixtures/structure.hcl | 5 - .../parser/test-fixtures/structure_basic.hcl | 5 - .../parser/test-fixtures/structure_empty.hcl | 1 - .../hcl/hcl/parser/test-fixtures/types.hcl | 7 - .../test-fixtures/unterminated_object.hcl | 2 - .../hashicorp/hcl/hcl/printer/nodes.go | 575 - .../hashicorp/hcl/hcl/printer/printer.go | 64 - .../hashicorp/hcl/hcl/printer/printer_test.go | 143 - .../hcl/hcl/printer/testdata/comment.golden | 36 - .../hcl/hcl/printer/testdata/comment.input | 37 - .../printer/testdata/comment_aligned.golden | 32 - .../printer/testdata/comment_aligned.input | 28 - .../testdata/comment_standalone.golden | 16 - .../printer/testdata/comment_standalone.input | 16 - .../hcl/printer/testdata/complexhcl.golden | 54 - .../hcl/hcl/printer/testdata/complexhcl.input | 53 - .../hcl/hcl/printer/testdata/list.golden | 27 - .../hcl/hcl/printer/testdata/list.input | 21 - .../hashicorp/hcl/hcl/scanner/scanner.go | 57 +- .../hashicorp/hcl/hcl/scanner/scanner_test.go | 536 - .../hashicorp/hcl/hcl/strconv/quote.go | 16 +- .../hashicorp/hcl/hcl/strconv/quote_test.go | 93 - .../hcl/hcl/test-fixtures/array_comment.hcl | 4 - .../hcl/hcl/test-fixtures/assign_colon.hcl | 6 - .../hcl/hcl/test-fixtures/assign_deep.hcl | 5 - .../hcl/hcl/test-fixtures/comment.hcl | 15 - .../hcl/hcl/test-fixtures/comment_single.hcl | 1 - .../hcl/hcl/test-fixtures/complex.hcl | 42 - .../hcl/hcl/test-fixtures/complex_key.hcl | 1 - .../hashicorp/hcl/hcl/test-fixtures/empty.hcl | 0 .../hashicorp/hcl/hcl/test-fixtures/list.hcl | 1 - .../hcl/hcl/test-fixtures/list_comma.hcl | 1 - .../hcl/hcl/test-fixtures/multiple.hcl | 2 - .../hashicorp/hcl/hcl/test-fixtures/old.hcl | 3 - .../hcl/hcl/test-fixtures/structure.hcl | 5 - .../hcl/hcl/test-fixtures/structure_basic.hcl | 5 - .../hcl/hcl/test-fixtures/structure_empty.hcl | 1 - .../hashicorp/hcl/hcl/test-fixtures/types.hcl | 7 - .../hashicorp/hcl/hcl/token/token.go | 63 +- .../hashicorp/hcl/hcl/token/token_test.go | 63 - vendor/github.com/hashicorp/hcl/hcl_test.go | 19 - .../hashicorp/hcl/json/parser/flatten.go | 6 + .../hashicorp/hcl/json/parser/parser.go | 16 + .../hashicorp/hcl/json/parser/parser_test.go | 338 - .../hcl/json/parser/test-fixtures/array.json | 4 - .../hcl/json/parser/test-fixtures/basic.json | 3 - .../hcl/json/parser/test-fixtures/object.json | 5 - .../hcl/json/parser/test-fixtures/types.json | 10 - .../hashicorp/hcl/json/scanner/scanner.go | 2 +- .../hcl/json/scanner/scanner_test.go | 363 - .../hcl/json/test-fixtures/array.json | 4 - .../hcl/json/test-fixtures/basic.json | 3 - .../hcl/json/test-fixtures/object.json | 5 - .../hcl/json/test-fixtures/types.json | 10 - .../hashicorp/hcl/json/token/token_test.go | 34 - vendor/github.com/hashicorp/hcl/lex.go | 17 +- vendor/github.com/hashicorp/hcl/lex_test.go | 37 - vendor/github.com/hashicorp/hcl/parse.go | 28 +- .../hashicorp/hcl/test-fixtures/basic.hcl | 2 - .../hashicorp/hcl/test-fixtures/basic.json | 4 - .../hcl/test-fixtures/basic_int_string.hcl | 1 - .../hcl/test-fixtures/basic_squish.hcl | 3 - .../hcl/test-fixtures/decode_policy.hcl | 15 - .../hcl/test-fixtures/decode_policy.json | 19 - .../hcl/test-fixtures/decode_tf_variable.hcl | 10 - .../hcl/test-fixtures/decode_tf_variable.json | 14 - .../hashicorp/hcl/test-fixtures/empty.hcl | 1 - .../hashicorp/hcl/test-fixtures/escape.hcl | 1 - .../hashicorp/hcl/test-fixtures/flat.hcl | 2 - .../hashicorp/hcl/test-fixtures/float.hcl | 1 - .../hashicorp/hcl/test-fixtures/float.json | 3 - .../hcl/test-fixtures/interpolate_escape.hcl | 1 - .../hashicorp/hcl/test-fixtures/multiline.hcl | 4 - .../hcl/test-fixtures/multiline.json | 3 - .../hcl/test-fixtures/multiline_bad.hcl | 4 - .../hcl/test-fixtures/multiline_no_eof.hcl | 5 - .../hcl/test-fixtures/multiline_no_marker.hcl | 1 - .../test-fixtures/nested_block_comment.hcl | 5 - .../hcl/test-fixtures/object_list.json | 15 - .../hcl/test-fixtures/scientific.hcl | 6 - .../hcl/test-fixtures/scientific.json | 8 - .../hcl/test-fixtures/slice_expand.hcl | 7 - .../hashicorp/hcl/test-fixtures/structure.hcl | 5 - .../hcl/test-fixtures/structure.json | 8 - .../hcl/test-fixtures/structure2.hcl | 9 - .../hcl/test-fixtures/structure2.json | 10 - .../hcl/test-fixtures/structure_flat.json | 8 - .../hcl/test-fixtures/structure_flatmap.hcl | 7 - .../hcl/test-fixtures/structure_list.hcl | 6 - .../hcl/test-fixtures/structure_list.json | 7 - .../test-fixtures/structure_list_deep.json | 16 - .../hcl/test-fixtures/structure_multi.hcl | 7 - .../hcl/test-fixtures/structure_multi.json | 11 - .../hcl/test-fixtures/terraform_heroku.hcl | 5 - .../hcl/test-fixtures/terraform_heroku.json | 6 - .../hashicorp/hcl/test-fixtures/tfvars.hcl | 3 - .../unterminated_block_comment.hcl | 2 - .../hcl/test-fixtures/unterminated_brace.hcl | 2 - .../inconshreveable/mousetrap/LICENSE | 13 + .../inconshreveable/mousetrap/README.md | 23 + .../inconshreveable/mousetrap/trap_others.go | 15 + .../inconshreveable/mousetrap/trap_windows.go | 98 + .../mousetrap/trap_windows_1.4.go | 46 + .../backoff}/LICENSE | 12 +- vendor/github.com/jpillora/backoff/README.md | 33 +- vendor/github.com/jpillora/backoff/backoff.go | 102 +- .../jpillora/backoff/backoff_test.go | 81 - vendor/github.com/kr/pretty/Readme | 9 - vendor/github.com/kr/pretty/diff.go | 158 - vendor/github.com/kr/pretty/diff_test.go | 74 - vendor/github.com/kr/pretty/example_test.go | 20 - vendor/github.com/kr/pretty/formatter.go | 337 - vendor/github.com/kr/pretty/formatter_test.go | 261 - vendor/github.com/kr/pretty/pretty.go | 98 - vendor/github.com/kr/pretty/zero.go | 41 - vendor/github.com/kr/text/Readme | 3 - vendor/github.com/kr/text/colwriter/Readme | 5 - vendor/github.com/kr/text/colwriter/column.go | 147 - .../kr/text/colwriter/column_test.go | 90 - vendor/github.com/kr/text/doc.go | 3 - vendor/github.com/kr/text/indent.go | 74 - vendor/github.com/kr/text/indent_test.go | 119 - vendor/github.com/kr/text/mc/Readme | 9 - vendor/github.com/kr/text/mc/mc.go | 62 - vendor/github.com/kr/text/wrap.go | 86 - vendor/github.com/kr/text/wrap_test.go | 62 - .../magiconair/properties/CHANGELOG.md | 23 +- .../magiconair/properties/README.md | 49 +- .../_third_party/gopkg.in/check.v1/LICENSE | 25 - .../_third_party/gopkg.in/check.v1/README.md | 20 - .../_third_party/gopkg.in/check.v1/TODO | 2 - .../gopkg.in/check.v1/benchmark.go | 163 - .../gopkg.in/check.v1/benchmark_test.go | 91 - .../gopkg.in/check.v1/bootstrap_test.go | 82 - .../_third_party/gopkg.in/check.v1/check.go | 945 -- .../gopkg.in/check.v1/check_test.go | 207 - .../gopkg.in/check.v1/checkers.go | 458 - .../gopkg.in/check.v1/checkers_test.go | 272 - .../gopkg.in/check.v1/export_test.go | 9 - .../gopkg.in/check.v1/fixture_test.go | 484 - .../gopkg.in/check.v1/foundation_test.go | 335 - .../_third_party/gopkg.in/check.v1/helpers.go | 231 - .../gopkg.in/check.v1/helpers_test.go | 519 - .../_third_party/gopkg.in/check.v1/printer.go | 168 - .../gopkg.in/check.v1/printer_test.go | 109 - .../_third_party/gopkg.in/check.v1/run.go | 175 - .../gopkg.in/check.v1/run_test.go | 419 - .../magiconair/properties/benchmark_test.go | 22 - .../magiconair/properties/decode.go | 23 +- .../magiconair/properties/decode_test.go | 295 - .../github.com/magiconair/properties/doc.go | 2 +- .../magiconair/properties/example_test.go | 93 - .../magiconair/properties/integrate.go | 4 + .../magiconair/properties/integrate_test.go | 70 - .../github.com/magiconair/properties/lex.go | 21 +- .../github.com/magiconair/properties/load.go | 173 +- .../magiconair/properties/load_test.go | 137 - .../magiconair/properties/parser.go | 2 +- .../magiconair/properties/properties.go | 74 +- .../magiconair/properties/properties_test.go | 906 -- .../magiconair/properties/rangecheck.go | 2 +- .../pretty/License => matryer/resync/LICENSE} | 10 +- vendor/github.com/matryer/resync/README.md | 4 +- vendor/github.com/matryer/resync/once_test.go | 35 - vendor/github.com/miekg/dns/AUTHORS | 1 + vendor/github.com/miekg/dns/CONTRIBUTORS | 9 + vendor/github.com/miekg/dns/COPYRIGHT | 9 + vendor/github.com/miekg/dns/LICENSE | 32 + vendor/github.com/miekg/dns/README.md | 160 + vendor/github.com/miekg/dns/client.go | 467 + vendor/github.com/miekg/dns/clientconfig.go | 131 + .../github.com/miekg/dns/compress_generate.go | 184 + vendor/github.com/miekg/dns/dane.go | 43 + vendor/github.com/miekg/dns/defaults.go | 282 + vendor/github.com/miekg/dns/dns.go | 104 + vendor/github.com/miekg/dns/dnssec.go | 720 + vendor/github.com/miekg/dns/dnssec_keygen.go | 156 + vendor/github.com/miekg/dns/dnssec_keyscan.go | 249 + vendor/github.com/miekg/dns/dnssec_privkey.go | 85 + vendor/github.com/miekg/dns/doc.go | 251 + vendor/github.com/miekg/dns/edns.go | 597 + vendor/github.com/miekg/dns/format.go | 87 + vendor/github.com/miekg/dns/generate.go | 159 + vendor/github.com/miekg/dns/labels.go | 171 + vendor/github.com/miekg/dns/msg.go | 1159 ++ vendor/github.com/miekg/dns/msg_generate.go | 349 + vendor/github.com/miekg/dns/msg_helpers.go | 633 + vendor/github.com/miekg/dns/nsecx.go | 106 + vendor/github.com/miekg/dns/privaterr.go | 149 + vendor/github.com/miekg/dns/rawmsg.go | 49 + vendor/github.com/miekg/dns/reverse.go | 38 + vendor/github.com/miekg/dns/sanitize.go | 84 + vendor/github.com/miekg/dns/scan.go | 981 ++ vendor/github.com/miekg/dns/scan_rr.go | 2184 +++ vendor/github.com/miekg/dns/scanner.go | 43 + vendor/github.com/miekg/dns/server.go | 734 + vendor/github.com/miekg/dns/sig0.go | 218 + vendor/github.com/miekg/dns/singleinflight.go | 57 + vendor/github.com/miekg/dns/smimea.go | 47 + vendor/github.com/miekg/dns/tlsa.go | 47 + vendor/github.com/miekg/dns/tsig.go | 383 + vendor/github.com/miekg/dns/types.go | 1287 ++ vendor/github.com/miekg/dns/types_generate.go | 271 + vendor/github.com/miekg/dns/udp.go | 34 + vendor/github.com/miekg/dns/udp_linux.go | 105 + vendor/github.com/miekg/dns/udp_other.go | 15 + vendor/github.com/miekg/dns/udp_windows.go | 29 + vendor/github.com/miekg/dns/update.go | 106 + vendor/github.com/miekg/dns/xfr.go | 244 + vendor/github.com/miekg/dns/zcompress.go | 119 + vendor/github.com/miekg/dns/zmsg.go | 3565 +++++ vendor/github.com/miekg/dns/ztypes.go | 857 ++ .../mitchellh/go-homedir/homedir.go | 7 +- .../mitchellh/go-homedir/homedir_test.go | 112 - .../mitchellh/mapstructure/decode_hooks.go | 5 +- .../mapstructure/decode_hooks_test.go | 229 - .../mitchellh/mapstructure/mapstructure.go | 106 +- .../mapstructure_benchmark_test.go | 279 - .../mapstructure/mapstructure_bugs_test.go | 47 - .../mapstructure_examples_test.go | 203 - .../mapstructure/mapstructure_test.go | 1047 -- .../pelletier/go-buffruneio/README.md | 62 + .../pelletier/go-buffruneio/buffruneio.go | 117 + vendor/github.com/pelletier/go-toml/LICENSE | 21 + vendor/github.com/pelletier/go-toml/README.md | 120 + vendor/github.com/pelletier/go-toml/clean.sh | 6 + vendor/github.com/pelletier/go-toml/doc.go | 250 + .../go-toml/example-crlf.toml} | 64 +- .../go-toml}/example.toml | 6 - .../pelletier/go-toml/keysparsing.go | 94 + vendor/github.com/pelletier/go-toml/lexer.go | 657 + .../github.com/pelletier/go-toml/marshal.go | 479 + .../pelletier/go-toml/marshal_test.toml | 38 + vendor/github.com/pelletier/go-toml/match.go | 234 + vendor/github.com/pelletier/go-toml/parser.go | 393 + .../github.com/pelletier/go-toml/position.go | 29 + vendor/github.com/pelletier/go-toml/query.go | 153 + .../pelletier/go-toml/querylexer.go | 356 + .../pelletier/go-toml/queryparser.go | 275 + vendor/github.com/pelletier/go-toml/test.sh | 82 + vendor/github.com/pelletier/go-toml/token.go | 140 + vendor/github.com/pelletier/go-toml/toml.go | 284 + .../pelletier/go-toml/tomltree_create.go | 135 + .../pelletier/go-toml/tomltree_write.go | 217 + vendor/github.com/pmezard/go-difflib/LICENSE | 27 + .../go-difflib/difflib/difflib_test.go | 426 - .../russross/blackfriday/LICENSE.txt | 29 - .../github.com/russross/blackfriday/README.md | 267 - .../github.com/russross/blackfriday/block.go | 1412 -- .../russross/blackfriday/block_test.go | 1621 --- .../github.com/russross/blackfriday/html.go | 949 -- .../github.com/russross/blackfriday/inline.go | 1148 -- .../russross/blackfriday/inline_test.go | 1194 -- .../github.com/russross/blackfriday/latex.go | 332 - .../russross/blackfriday/markdown.go | 923 -- .../russross/blackfriday/ref_test.go | 128 - .../russross/blackfriday/smartypants.go | 400 - .../testdata/Amps and angle encoding.html | 17 - .../testdata/Amps and angle encoding.text | 21 - .../blackfriday/testdata/Auto links.html | 18 - .../blackfriday/testdata/Auto links.text | 13 - .../testdata/Backslash escapes.html | 123 - .../testdata/Backslash escapes.text | 126 - .../Blockquotes with code blocks.html | 15 - .../Blockquotes with code blocks.text | 11 - .../blackfriday/testdata/Code Blocks.html | 18 - .../blackfriday/testdata/Code Blocks.text | 14 - .../blackfriday/testdata/Code Spans.html | 5 - .../blackfriday/testdata/Code Spans.text | 6 - ...like lines no empty line before block.html | 14 - ...like lines no empty line before block.text | 8 - ...apped paragraphs with list-like lines.html | 8 - ...apped paragraphs with list-like lines.text | 8 - .../testdata/Horizontal rules.html | 71 - .../testdata/Horizontal rules.text | 67 - .../testdata/Inline HTML (Advanced).html | 15 - .../testdata/Inline HTML (Advanced).text | 15 - .../testdata/Inline HTML (Simple).html | 72 - .../testdata/Inline HTML (Simple).text | 69 - .../testdata/Inline HTML comments.html | 13 - .../testdata/Inline HTML comments.text | 13 - .../testdata/Links, inline style.html | 11 - .../testdata/Links, inline style.text | 12 - .../testdata/Links, reference style.html | 52 - .../testdata/Links, reference style.text | 71 - .../testdata/Links, shortcut references.html | 9 - .../testdata/Links, shortcut references.text | 20 - .../testdata/Literal quotes in titles.html | 3 - .../testdata/Literal quotes in titles.text | 7 - .../Markdown Documentation - Basics.html | 314 - .../Markdown Documentation - Basics.text | 306 - .../Markdown Documentation - Syntax.html | 946 -- .../Markdown Documentation - Syntax.text | 888 -- .../testdata/Nested blockquotes.html | 9 - .../testdata/Nested blockquotes.text | 5 - .../testdata/Ordered and unordered lists.html | 166 - .../testdata/Ordered and unordered lists.text | 131 - .../testdata/Strong and em together.html | 7 - .../testdata/Strong and em together.text | 7 - .../russross/blackfriday/testdata/Tabs.html | 26 - .../russross/blackfriday/testdata/Tabs.text | 21 - .../blackfriday/testdata/Tidyness.html | 9 - .../blackfriday/testdata/Tidyness.text | 5 - .../ryszard/goskiplist/skiplist/skiplist.go | 635 - .../goskiplist/skiplist/skiplist_test.go | 924 -- .../shurcooL/sanitized_anchor_name/README.md | 31 - .../shurcooL/sanitized_anchor_name/main.go | 29 - .../sanitized_anchor_name/main_test.go | 35 - vendor/github.com/spf13/afero/LICENSE.txt | 174 + vendor/github.com/spf13/afero/README.md | 449 + vendor/github.com/spf13/afero/afero.go | 108 + vendor/github.com/spf13/afero/appveyor.yml | 15 + vendor/github.com/spf13/afero/basepath.go | 145 + .../github.com/spf13/afero/cacheOnReadFs.go | 295 + vendor/github.com/spf13/afero/const_bsds.go | 22 + .../github.com/spf13/afero/const_win_unix.go | 25 + .../github.com/spf13/afero/copyOnWriteFs.go | 253 + vendor/github.com/spf13/afero/httpFs.go | 110 + vendor/github.com/spf13/afero/ioutil.go | 230 + vendor/github.com/spf13/afero/mem/dir.go | 37 + vendor/github.com/spf13/afero/mem/dirmap.go | 43 + vendor/github.com/spf13/afero/mem/file.go | 285 + vendor/github.com/spf13/afero/memmap.go | 361 + .../cobra/main.go => afero/memradix.go} | 10 +- vendor/github.com/spf13/afero/os.go | 94 + vendor/github.com/spf13/afero/path.go | 108 + vendor/github.com/spf13/afero/readonlyfs.go | 70 + vendor/github.com/spf13/afero/regexpfs.go | 214 + vendor/github.com/spf13/afero/unionFile.go | 274 + vendor/github.com/spf13/afero/util.go | 331 + vendor/github.com/spf13/cast/Makefile | 38 + vendor/github.com/spf13/cast/README.md | 3 + vendor/github.com/spf13/cast/cast.go | 86 + vendor/github.com/spf13/cast/cast_test.go | 164 - vendor/github.com/spf13/cast/caste.go | 856 +- vendor/github.com/spf13/cobra/README.md | 52 +- .../spf13/cobra/bash_completions.go | 167 +- .../spf13/cobra/bash_completions.md | 63 +- .../spf13/cobra/bash_completions_test.go | 95 - vendor/github.com/spf13/cobra/cobra.go | 58 +- .../github.com/spf13/cobra/cobra/cmd/add.go | 128 - .../spf13/cobra/cobra/cmd/helpers.go | 347 - .../spf13/cobra/cobra/cmd/helpers_test.go | 40 - .../github.com/spf13/cobra/cobra/cmd/init.go | 226 - .../spf13/cobra/cobra/cmd/licenses.go | 1133 -- .../github.com/spf13/cobra/cobra/cmd/root.go | 84 - vendor/github.com/spf13/cobra/cobra_test.go | 1169 -- vendor/github.com/spf13/cobra/command.go | 426 +- .../github.com/spf13/cobra/command_notwin.go | 2 +- vendor/github.com/spf13/cobra/command_test.go | 114 - vendor/github.com/spf13/cobra/doc/cmd_test.go | 145 - vendor/github.com/spf13/cobra/doc/man_docs.go | 218 - vendor/github.com/spf13/cobra/doc/man_docs.md | 26 - .../spf13/cobra/doc/man_docs_test.go | 97 - .../spf13/cobra/doc/man_examples_test.go | 35 - vendor/github.com/spf13/cobra/doc/md_docs.go | 175 - vendor/github.com/spf13/cobra/doc/md_docs.md | 104 - .../spf13/cobra/doc/md_docs_test.go | 88 - vendor/github.com/spf13/cobra/doc/util.go | 38 - .../spf13/jwalterweatherman/README.md | 33 +- .../jwalterweatherman/default_notepad.go | 113 + .../spf13/jwalterweatherman/jww_test.go | 56 - .../spf13/jwalterweatherman/log_counter.go | 56 + .../spf13/jwalterweatherman/notepad.go | 195 + .../thatswhyyoualwaysleaveanote.go | 198 - vendor/github.com/spf13/pflag/README.md | 33 +- vendor/github.com/spf13/pflag/bool.go | 7 +- vendor/github.com/spf13/pflag/bool_slice.go | 147 + vendor/github.com/spf13/pflag/bool_test.go | 180 - vendor/github.com/spf13/pflag/count.go | 7 +- vendor/github.com/spf13/pflag/count_test.go | 55 - vendor/github.com/spf13/pflag/example_test.go | 77 - vendor/github.com/spf13/pflag/export_test.go | 29 - vendor/github.com/spf13/pflag/flag.go | 291 +- vendor/github.com/spf13/pflag/flag_test.go | 913 -- vendor/github.com/spf13/pflag/float32.go | 7 +- vendor/github.com/spf13/pflag/float64.go | 7 +- vendor/github.com/spf13/pflag/golangflag.go | 3 - .../github.com/spf13/pflag/golangflag_test.go | 39 - vendor/github.com/spf13/pflag/int.go | 7 +- vendor/github.com/spf13/pflag/int32.go | 7 +- vendor/github.com/spf13/pflag/int64.go | 7 +- vendor/github.com/spf13/pflag/int8.go | 7 +- .../github.com/spf13/pflag/int_slice_test.go | 162 - vendor/github.com/spf13/pflag/ip.go | 2 - vendor/github.com/spf13/pflag/ip_slice.go | 148 + vendor/github.com/spf13/pflag/ip_test.go | 63 - vendor/github.com/spf13/pflag/ipnet.go | 2 - vendor/github.com/spf13/pflag/ipnet_test.go | 70 - vendor/github.com/spf13/pflag/string.go | 4 +- vendor/github.com/spf13/pflag/string_array.go | 103 + vendor/github.com/spf13/pflag/string_slice.go | 36 +- .../spf13/pflag/string_slice_test.go | 161 - vendor/github.com/spf13/pflag/uint.go | 7 +- vendor/github.com/spf13/pflag/uint16.go | 9 +- vendor/github.com/spf13/pflag/uint32.go | 11 +- vendor/github.com/spf13/pflag/uint64.go | 7 +- vendor/github.com/spf13/pflag/uint8.go | 7 +- vendor/github.com/spf13/pflag/uint_slice.go | 126 + vendor/github.com/spf13/pflag/verify/all.sh | 69 - vendor/github.com/spf13/pflag/verify/gofmt.sh | 19 - .../github.com/spf13/pflag/verify/golint.sh | 15 - vendor/github.com/spf13/viper/README.md | 35 +- vendor/github.com/spf13/viper/flags_test.go | 66 - vendor/github.com/spf13/viper/nohup.out | 1 + .../github.com/spf13/viper/remote/remote.go | 77 - vendor/github.com/spf13/viper/util.go | 129 +- vendor/github.com/spf13/viper/viper.go | 852 +- vendor/github.com/spf13/viper/viper_test.go | 885 -- .../square/go-jose/asymmetric_test.go | 431 - .../square/go-jose/cipher/cbc_hmac_test.go | 498 - .../square/go-jose/cipher/concat_kdf_test.go | 150 - .../square/go-jose/cipher/ecdh_es_test.go | 98 - .../square/go-jose/cipher/key_wrap_test.go | 133 - .../github.com/square/go-jose/crypter_test.go | 784 - vendor/github.com/square/go-jose/doc_test.go | 226 - .../square/go-jose/encoding_test.go | 173 - .../square/go-jose/jose-util/README.md | 59 - .../square/go-jose/jose-util/jose-util.t | 88 - .../square/go-jose/jose-util/main.go | 300 - vendor/github.com/square/go-jose/jwe_test.go | 537 - vendor/github.com/square/go-jose/jwk_test.go | 576 - vendor/github.com/square/go-jose/jws_test.go | 302 - .../github.com/square/go-jose/signing_test.go | 423 - .../square/go-jose/symmetric_test.go | 131 - .../github.com/square/go-jose/utils_test.go | 225 - .../github.com/steveyen/gtreap/treap_test.go | 259 - vendor/github.com/stretchr/testify/LICENSE | 22 + .../testify/assert/assertion_forward.go | 173 +- .../stretchr/testify/assert/assertions.go | 183 +- .../testify/assert/assertions_test.go | 1122 -- .../testify/assert/forward_assertions_test.go | 611 - .../testify/assert/http_assertions.go | 2 +- .../testify/assert/http_assertions_test.go | 86 - .../syndtr/goleveldb/leveldb/batch.go | 261 - .../syndtr/goleveldb/leveldb/batch_test.go | 120 - .../syndtr/goleveldb/leveldb/bench2_test.go | 58 - .../syndtr/goleveldb/leveldb/bench_test.go | 464 - .../goleveldb/leveldb/cache/bench2_test.go | 30 - .../syndtr/goleveldb/leveldb/cache/cache.go | 684 - .../goleveldb/leveldb/cache/cache_test.go | 553 - .../syndtr/goleveldb/leveldb/cache/lru.go | 195 - .../syndtr/goleveldb/leveldb/comparer.go | 75 - .../leveldb/comparer/bytes_comparer.go | 51 - .../goleveldb/leveldb/comparer/comparer.go | 57 - .../syndtr/goleveldb/leveldb/corrupt_test.go | 496 - .../github.com/syndtr/goleveldb/leveldb/db.go | 1091 -- .../syndtr/goleveldb/leveldb/db_compaction.go | 826 -- .../syndtr/goleveldb/leveldb/db_iter.go | 360 - .../syndtr/goleveldb/leveldb/db_snapshot.go | 183 - .../syndtr/goleveldb/leveldb/db_state.go | 219 - .../syndtr/goleveldb/leveldb/db_test.go | 2803 ---- .../goleveldb/leveldb/db_transaction.go | 289 - .../syndtr/goleveldb/leveldb/db_util.go | 102 - .../syndtr/goleveldb/leveldb/db_write.go | 358 - .../syndtr/goleveldb/leveldb/doc.go | 90 - .../syndtr/goleveldb/leveldb/errors.go | 19 - .../syndtr/goleveldb/leveldb/errors/errors.go | 78 - .../syndtr/goleveldb/leveldb/external_test.go | 117 - .../syndtr/goleveldb/leveldb/filter.go | 31 - .../syndtr/goleveldb/leveldb/filter/bloom.go | 116 - .../goleveldb/leveldb/filter/bloom_test.go | 142 - .../syndtr/goleveldb/leveldb/filter/filter.go | 60 - .../goleveldb/leveldb/iterator/array_iter.go | 184 - .../leveldb/iterator/array_iter_test.go | 30 - .../leveldb/iterator/indexed_iter.go | 242 - .../leveldb/iterator/indexed_iter_test.go | 83 - .../syndtr/goleveldb/leveldb/iterator/iter.go | 131 - .../leveldb/iterator/iter_suite_test.go | 11 - .../goleveldb/leveldb/iterator/merged_iter.go | 304 - .../leveldb/iterator/merged_iter_test.go | 60 - .../goleveldb/leveldb/journal/journal.go | 521 - .../goleveldb/leveldb/journal/journal_test.go | 818 -- .../syndtr/goleveldb/leveldb/key.go | 147 - .../syndtr/goleveldb/leveldb/key_test.go | 133 - .../goleveldb/leveldb/leveldb_suite_test.go | 11 - .../goleveldb/leveldb/memdb/bench_test.go | 75 - .../syndtr/goleveldb/leveldb/memdb/memdb.go | 471 - .../leveldb/memdb/memdb_suite_test.go | 11 - .../goleveldb/leveldb/memdb/memdb_test.go | 135 - .../syndtr/goleveldb/leveldb/opt/options.go | 660 - .../syndtr/goleveldb/leveldb/options.go | 107 - .../syndtr/goleveldb/leveldb/session.go | 207 - .../goleveldb/leveldb/session_compaction.go | 302 - .../goleveldb/leveldb/session_record.go | 323 - .../goleveldb/leveldb/session_record_test.go | 62 - .../syndtr/goleveldb/leveldb/session_util.go | 252 - .../goleveldb/leveldb/storage/file_storage.go | 583 - .../leveldb/storage/file_storage_plan9.go | 65 - .../leveldb/storage/file_storage_solaris.go | 81 - .../leveldb/storage/file_storage_test.go | 176 - .../leveldb/storage/file_storage_unix.go | 86 - .../leveldb/storage/file_storage_windows.go | 78 - .../goleveldb/leveldb/storage/mem_storage.go | 218 - .../leveldb/storage/mem_storage_test.go | 65 - .../goleveldb/leveldb/storage/storage.go | 177 - .../syndtr/goleveldb/leveldb/table.go | 529 - .../goleveldb/leveldb/table/block_test.go | 139 - .../syndtr/goleveldb/leveldb/table/reader.go | 1107 -- .../syndtr/goleveldb/leveldb/table/table.go | 177 - .../leveldb/table/table_suite_test.go | 11 - .../goleveldb/leveldb/table/table_test.go | 123 - .../syndtr/goleveldb/leveldb/table/writer.go | 375 - .../syndtr/goleveldb/leveldb/testutil/db.go | 222 - .../goleveldb/leveldb/testutil/ginkgo.go | 21 - .../syndtr/goleveldb/leveldb/testutil/iter.go | 327 - .../syndtr/goleveldb/leveldb/testutil/kv.go | 352 - .../goleveldb/leveldb/testutil/kvtest.go | 211 - .../goleveldb/leveldb/testutil/storage.go | 694 - .../syndtr/goleveldb/leveldb/testutil/util.go | 171 - .../syndtr/goleveldb/leveldb/testutil_test.go | 91 - .../syndtr/goleveldb/leveldb/util.go | 91 - .../syndtr/goleveldb/leveldb/util/buffer.go | 293 - .../goleveldb/leveldb/util/buffer_pool.go | 239 - .../goleveldb/leveldb/util/buffer_test.go | 369 - .../syndtr/goleveldb/leveldb/util/crc32.go | 30 - .../syndtr/goleveldb/leveldb/util/hash.go | 48 - .../syndtr/goleveldb/leveldb/util/pool.go | 21 - .../goleveldb/leveldb/util/pool_legacy.go | 33 - .../syndtr/goleveldb/leveldb/util/range.go | 32 - .../syndtr/goleveldb/leveldb/util/util.go | 73 - .../syndtr/goleveldb/leveldb/version.go | 519 - .../syndtr/goleveldb/leveldb/version_test.go | 181 - vendor/github.com/willf/bitset/Makefile | 109 - vendor/github.com/willf/bitset/README.md | 65 - vendor/github.com/willf/bitset/RELEASE | 1 - vendor/github.com/willf/bitset/VERSION | 1 - vendor/github.com/willf/bitset/bitset.go | 650 - vendor/github.com/willf/bitset/bitset_test.go | 837 -- vendor/github.com/willf/bitset/popcnt.go | 53 - .../github.com/willf/bitset/popcnt_amd64.go | 67 - vendor/github.com/willf/bitset/popcnt_amd64.s | 103 - .../github.com/willf/bitset/popcnt_generic.go | 23 - vendor/github.com/willf/bitset/popcnt_test.go | 58 - .../{kr/text/License => xenolf/lego/LICENSE} | 12 +- .../github.com/xenolf/lego/acme/challenges.go | 16 + vendor/github.com/xenolf/lego/acme/client.go | 493 +- .../xenolf/lego/acme/client_test.go | 196 - vendor/github.com/xenolf/lego/acme/crypto.go | 137 +- .../xenolf/lego/acme/crypto_test.go | 92 - .../xenolf/lego/acme/dns_challenge.go | 304 + .../xenolf/lego/acme/dns_challenge_manual.go | 53 + vendor/github.com/xenolf/lego/acme/error.go | 29 +- vendor/github.com/xenolf/lego/acme/http.go | 61 +- .../xenolf/lego/acme/http_challenge.go | 65 +- .../xenolf/lego/acme/http_challenge_server.go | 79 + .../xenolf/lego/acme/http_challenge_test.go | 56 - .../github.com/xenolf/lego/acme/http_test.go | 100 - vendor/github.com/xenolf/lego/acme/jws.go | 112 +- .../github.com/xenolf/lego/acme/messages.go | 52 +- .../github.com/xenolf/lego/acme/provider.go | 28 + .../xenolf/lego/acme/tls_sni_challenge.go | 60 +- .../lego/acme/tls_sni_challenge_server.go | 62 + .../lego/acme/tls_sni_challenge_test.go | 64 - vendor/github.com/xenolf/lego/acme/utils.go | 29 + .../bitset => golang.org/x/crypto}/LICENSE | 2 +- vendor/golang.org/x/crypto/PATENTS | 22 + vendor/golang.org/x/crypto/ocsp/ocsp.go | 196 +- vendor/golang.org/x/crypto/ocsp/ocsp_test.go | 584 - vendor/golang.org/x/crypto/sha3/doc.go | 66 - vendor/golang.org/x/crypto/sha3/hashes.go | 65 - vendor/golang.org/x/crypto/sha3/keccakf.go | 410 - vendor/golang.org/x/crypto/sha3/register.go | 18 - vendor/golang.org/x/crypto/sha3/sha3.go | 193 - vendor/golang.org/x/crypto/sha3/sha3_test.go | 306 - vendor/golang.org/x/crypto/sha3/shake.go | 60 - .../sha3/testdata/keccakKats.json.deflate | Bin 521342 -> 0 bytes vendor/golang.org/x/crypto/sha3/xor.go | 16 - .../golang.org/x/crypto/sha3/xor_generic.go | 28 - .../golang.org/x/crypto/sha3/xor_unaligned.go | 58 - .../x/crypto/ssh/terminal/terminal.go | 892 -- .../x/crypto/ssh/terminal/terminal_test.go | 269 - .../golang.org/x/crypto/ssh/terminal/util.go | 128 - .../x/crypto/ssh/terminal/util_bsd.go | 12 - .../x/crypto/ssh/terminal/util_linux.go | 11 - .../x/crypto/ssh/terminal/util_windows.go | 174 - .../snappy => golang.org/x/net}/LICENSE | 2 +- vendor/golang.org/x/net/PATENTS | 22 + vendor/golang.org/x/net/context/context.go | 156 + vendor/golang.org/x/net/context/go17.go | 72 + vendor/golang.org/x/net/context/pre_go17.go | 300 + vendor/golang.org/x/net/publicsuffix/gen.go | 713 + vendor/golang.org/x/net/publicsuffix/list.go | 135 + vendor/golang.org/x/net/publicsuffix/table.go | 9093 ++++++++++++ .../cznic/b => golang.org/x/sys}/LICENSE | 4 +- vendor/golang.org/x/sys/PATENTS | 22 + vendor/golang.org/x/sys/unix/asm_darwin_386.s | 29 + .../golang.org/x/sys/unix/asm_darwin_amd64.s | 29 + vendor/golang.org/x/sys/unix/asm_darwin_arm.s | 30 + .../golang.org/x/sys/unix/asm_darwin_arm64.s | 30 + .../x/sys/unix/asm_dragonfly_amd64.s | 29 + .../golang.org/x/sys/unix/asm_freebsd_386.s | 29 + .../golang.org/x/sys/unix/asm_freebsd_amd64.s | 29 + .../golang.org/x/sys/unix/asm_freebsd_arm.s | 29 + vendor/golang.org/x/sys/unix/asm_linux_386.s | 35 + .../golang.org/x/sys/unix/asm_linux_amd64.s | 29 + vendor/golang.org/x/sys/unix/asm_linux_arm.s | 29 + .../golang.org/x/sys/unix/asm_linux_arm64.s | 24 + .../golang.org/x/sys/unix/asm_linux_mips64x.s | 28 + .../golang.org/x/sys/unix/asm_linux_mipsx.s | 31 + .../golang.org/x/sys/unix/asm_linux_ppc64x.s | 28 + .../golang.org/x/sys/unix/asm_linux_s390x.s | 28 + vendor/golang.org/x/sys/unix/asm_netbsd_386.s | 29 + .../golang.org/x/sys/unix/asm_netbsd_amd64.s | 29 + vendor/golang.org/x/sys/unix/asm_netbsd_arm.s | 29 + .../golang.org/x/sys/unix/asm_openbsd_386.s | 29 + .../golang.org/x/sys/unix/asm_openbsd_amd64.s | 29 + .../golang.org/x/sys/unix/asm_solaris_amd64.s | 17 + .../golang.org/x/sys/unix/bluetooth_linux.go | 35 + vendor/golang.org/x/sys/unix/constants.go | 13 + vendor/golang.org/x/sys/unix/dirent.go | 102 + vendor/golang.org/x/sys/unix/endian_big.go | 9 + vendor/golang.org/x/sys/unix/endian_little.go | 9 + vendor/golang.org/x/sys/unix/env_unix.go | 27 + vendor/golang.org/x/sys/unix/env_unset.go | 14 + vendor/golang.org/x/sys/unix/flock.go | 24 + .../x/sys/unix/flock_linux_32bit.go | 13 + vendor/golang.org/x/sys/unix/gccgo.go | 46 + vendor/golang.org/x/sys/unix/gccgo_c.c | 41 + .../x/sys/unix/gccgo_linux_amd64.go | 20 + .../x/sys/unix/gccgo_linux_sparc64.go | 20 + vendor/golang.org/x/sys/unix/mkall.sh | 294 + vendor/golang.org/x/sys/unix/mkerrors.sh | 499 + vendor/golang.org/x/sys/unix/mkpost.go | 62 + vendor/golang.org/x/sys/unix/mksyscall.pl | 318 + .../x/sys/unix/mksyscall_solaris.pl | 289 + .../golang.org/x/sys/unix/mksysctl_openbsd.pl | 264 + .../golang.org/x/sys/unix/mksysnum_darwin.pl | 39 + .../x/sys/unix/mksysnum_dragonfly.pl | 50 + .../golang.org/x/sys/unix/mksysnum_freebsd.pl | 63 + .../golang.org/x/sys/unix/mksysnum_linux.pl | 78 + .../golang.org/x/sys/unix/mksysnum_netbsd.pl | 58 + .../golang.org/x/sys/unix/mksysnum_openbsd.pl | 50 + .../golang.org/x/sys/unix/openbsd_pledge.go | 38 + vendor/golang.org/x/sys/unix/race.go | 30 + vendor/golang.org/x/sys/unix/race0.go | 25 + .../golang.org/x/sys/unix/sockcmsg_linux.go | 36 + vendor/golang.org/x/sys/unix/sockcmsg_unix.go | 103 + vendor/golang.org/x/sys/unix/str.go | 26 + vendor/golang.org/x/sys/unix/syscall.go | 69 + vendor/golang.org/x/sys/unix/syscall_bsd.go | 614 + .../golang.org/x/sys/unix/syscall_darwin.go | 493 + .../x/sys/unix/syscall_darwin_386.go | 77 + .../x/sys/unix/syscall_darwin_amd64.go | 79 + .../x/sys/unix/syscall_darwin_arm.go | 71 + .../x/sys/unix/syscall_darwin_arm64.go | 77 + .../x/sys/unix/syscall_dragonfly.go | 402 + .../x/sys/unix/syscall_dragonfly_amd64.go | 61 + .../golang.org/x/sys/unix/syscall_freebsd.go | 666 + .../x/sys/unix/syscall_freebsd_386.go | 61 + .../x/sys/unix/syscall_freebsd_amd64.go | 61 + .../x/sys/unix/syscall_freebsd_arm.go | 61 + vendor/golang.org/x/sys/unix/syscall_linux.go | 1297 ++ .../x/sys/unix/syscall_linux_386.go | 399 + .../x/sys/unix/syscall_linux_amd64.go | 152 + .../x/sys/unix/syscall_linux_amd64_gc.go | 13 + .../x/sys/unix/syscall_linux_arm.go | 263 + .../x/sys/unix/syscall_linux_arm64.go | 190 + .../x/sys/unix/syscall_linux_mips64x.go | 208 + .../x/sys/unix/syscall_linux_mipsx.go | 239 + .../x/sys/unix/syscall_linux_ppc64x.go | 135 + .../x/sys/unix/syscall_linux_s390x.go | 328 + .../x/sys/unix/syscall_linux_sparc64.go | 169 + .../golang.org/x/sys/unix/syscall_netbsd.go | 476 + .../x/sys/unix/syscall_netbsd_386.go | 42 + .../x/sys/unix/syscall_netbsd_amd64.go | 42 + .../x/sys/unix/syscall_netbsd_arm.go | 42 + .../golang.org/x/sys/unix/syscall_no_getwd.go | 11 + .../golang.org/x/sys/unix/syscall_openbsd.go | 287 + .../x/sys/unix/syscall_openbsd_386.go | 42 + .../x/sys/unix/syscall_openbsd_amd64.go | 42 + .../golang.org/x/sys/unix/syscall_solaris.go | 713 + .../x/sys/unix/syscall_solaris_amd64.go | 35 + vendor/golang.org/x/sys/unix/syscall_unix.go | 292 + .../golang.org/x/sys/unix/syscall_unix_gc.go | 15 + vendor/golang.org/x/sys/unix/types_darwin.go | 250 + .../golang.org/x/sys/unix/types_dragonfly.go | 242 + vendor/golang.org/x/sys/unix/types_freebsd.go | 353 + vendor/golang.org/x/sys/unix/types_linux.go | 469 + vendor/golang.org/x/sys/unix/types_netbsd.go | 232 + vendor/golang.org/x/sys/unix/types_openbsd.go | 244 + vendor/golang.org/x/sys/unix/types_solaris.go | 262 + .../x/sys/unix/zerrors_darwin_386.go | 1576 ++ .../x/sys/unix/zerrors_darwin_amd64.go | 1576 ++ .../x/sys/unix/zerrors_darwin_arm.go | 1293 ++ .../x/sys/unix/zerrors_darwin_arm64.go | 1576 ++ .../x/sys/unix/zerrors_dragonfly_amd64.go | 1530 ++ .../x/sys/unix/zerrors_freebsd_386.go | 1743 +++ .../x/sys/unix/zerrors_freebsd_amd64.go | 1748 +++ .../x/sys/unix/zerrors_freebsd_arm.go | 1729 +++ .../x/sys/unix/zerrors_linux_386.go | 1887 +++ .../x/sys/unix/zerrors_linux_amd64.go | 1895 +++ .../x/sys/unix/zerrors_linux_arm.go | 1815 +++ .../x/sys/unix/zerrors_linux_arm64.go | 1967 +++ .../x/sys/unix/zerrors_linux_mips.go | 1869 +++ .../x/sys/unix/zerrors_linux_mips64.go | 1965 +++ .../x/sys/unix/zerrors_linux_mips64le.go | 1965 +++ .../x/sys/unix/zerrors_linux_mipsle.go | 2066 +++ .../x/sys/unix/zerrors_linux_ppc64.go | 2040 +++ .../x/sys/unix/zerrors_linux_ppc64le.go | 2036 +++ .../x/sys/unix/zerrors_linux_s390x.go | 2093 +++ .../x/sys/unix/zerrors_linux_sparc64.go | 2142 +++ .../x/sys/unix/zerrors_netbsd_386.go | 1712 +++ .../x/sys/unix/zerrors_netbsd_amd64.go | 1702 +++ .../x/sys/unix/zerrors_netbsd_arm.go | 1688 +++ .../x/sys/unix/zerrors_openbsd_386.go | 1584 ++ .../x/sys/unix/zerrors_openbsd_amd64.go | 1583 ++ .../x/sys/unix/zerrors_solaris_amd64.go | 1436 ++ .../x/sys/unix/zsyscall_darwin_386.go | 1394 ++ .../x/sys/unix/zsyscall_darwin_amd64.go | 1409 ++ .../x/sys/unix/zsyscall_darwin_arm.go | 1394 ++ .../x/sys/unix/zsyscall_darwin_arm64.go | 1394 ++ .../x/sys/unix/zsyscall_dragonfly_amd64.go | 1382 ++ .../x/sys/unix/zsyscall_freebsd_386.go | 1617 +++ .../x/sys/unix/zsyscall_freebsd_amd64.go | 1617 +++ .../x/sys/unix/zsyscall_freebsd_arm.go | 1617 +++ .../x/sys/unix/zsyscall_linux_386.go | 1649 +++ .../x/sys/unix/zsyscall_linux_amd64.go | 1842 +++ .../x/sys/unix/zsyscall_linux_arm.go | 1751 +++ .../x/sys/unix/zsyscall_linux_arm64.go | 1725 +++ .../x/sys/unix/zsyscall_linux_mips.go | 1807 +++ .../x/sys/unix/zsyscall_linux_mips64.go | 1791 +++ .../x/sys/unix/zsyscall_linux_mips64le.go | 1791 +++ .../x/sys/unix/zsyscall_linux_mipsle.go | 1807 +++ .../x/sys/unix/zsyscall_linux_ppc64.go | 1853 +++ .../x/sys/unix/zsyscall_linux_ppc64le.go | 1853 +++ .../x/sys/unix/zsyscall_linux_s390x.go | 1633 +++ .../x/sys/unix/zsyscall_linux_sparc64.go | 1822 +++ .../x/sys/unix/zsyscall_netbsd_386.go | 1299 ++ .../x/sys/unix/zsyscall_netbsd_amd64.go | 1299 ++ .../x/sys/unix/zsyscall_netbsd_arm.go | 1299 ++ .../x/sys/unix/zsyscall_openbsd_386.go | 1357 ++ .../x/sys/unix/zsyscall_openbsd_amd64.go | 1357 ++ .../x/sys/unix/zsyscall_solaris_amd64.go | 1562 ++ .../golang.org/x/sys/unix/zsysctl_openbsd.go | 270 + .../x/sys/unix/zsysnum_darwin_386.go | 398 + .../x/sys/unix/zsysnum_darwin_amd64.go | 398 + .../x/sys/unix/zsysnum_darwin_arm.go | 358 + .../x/sys/unix/zsysnum_darwin_arm64.go | 398 + .../x/sys/unix/zsysnum_dragonfly_amd64.go | 304 + .../x/sys/unix/zsysnum_freebsd_386.go | 351 + .../x/sys/unix/zsysnum_freebsd_amd64.go | 351 + .../x/sys/unix/zsysnum_freebsd_arm.go | 351 + .../x/sys/unix/zsysnum_linux_386.go | 388 + .../x/sys/unix/zsysnum_linux_amd64.go | 341 + .../x/sys/unix/zsysnum_linux_arm.go | 361 + .../x/sys/unix/zsysnum_linux_arm64.go | 285 + .../x/sys/unix/zsysnum_linux_mips.go | 374 + .../x/sys/unix/zsysnum_linux_mips64.go | 334 + .../x/sys/unix/zsysnum_linux_mips64le.go | 334 + .../x/sys/unix/zsysnum_linux_mipsle.go | 374 + .../x/sys/unix/zsysnum_linux_ppc64.go | 369 + .../x/sys/unix/zsysnum_linux_ppc64le.go | 369 + .../x/sys/unix/zsysnum_linux_s390x.go | 331 + .../x/sys/unix/zsysnum_linux_sparc64.go | 348 + .../x/sys/unix/zsysnum_netbsd_386.go | 273 + .../x/sys/unix/zsysnum_netbsd_amd64.go | 273 + .../x/sys/unix/zsysnum_netbsd_arm.go | 273 + .../x/sys/unix/zsysnum_openbsd_386.go | 207 + .../x/sys/unix/zsysnum_openbsd_amd64.go | 207 + .../x/sys/unix/zsysnum_solaris_amd64.go | 13 + .../x/sys/unix/ztypes_darwin_386.go | 447 + .../x/sys/unix/ztypes_darwin_amd64.go | 462 + .../x/sys/unix/ztypes_darwin_arm.go | 449 + .../x/sys/unix/ztypes_darwin_arm64.go | 457 + .../x/sys/unix/ztypes_dragonfly_amd64.go | 443 + .../x/sys/unix/ztypes_freebsd_386.go | 502 + .../x/sys/unix/ztypes_freebsd_amd64.go | 505 + .../x/sys/unix/ztypes_freebsd_arm.go | 497 + .../golang.org/x/sys/unix/ztypes_linux_386.go | 653 + .../x/sys/unix/ztypes_linux_amd64.go | 671 + .../golang.org/x/sys/unix/ztypes_linux_arm.go | 642 + .../x/sys/unix/ztypes_linux_arm64.go | 650 + .../x/sys/unix/ztypes_linux_mips.go | 660 + .../x/sys/unix/ztypes_linux_mips64.go | 653 + .../x/sys/unix/ztypes_linux_mips64le.go | 653 + .../x/sys/unix/ztypes_linux_mipsle.go | 660 + .../x/sys/unix/ztypes_linux_ppc64.go | 660 + .../x/sys/unix/ztypes_linux_ppc64le.go | 660 + .../x/sys/unix/ztypes_linux_s390x.go | 675 + .../x/sys/unix/ztypes_linux_sparc64.go | 666 + .../x/sys/unix/ztypes_netbsd_386.go | 396 + .../x/sys/unix/ztypes_netbsd_amd64.go | 403 + .../x/sys/unix/ztypes_netbsd_arm.go | 401 + .../x/sys/unix/ztypes_openbsd_386.go | 441 + .../x/sys/unix/ztypes_openbsd_amd64.go | 448 + .../x/sys/unix/ztypes_solaris_amd64.go | 423 + vendor/golang.org/x/text/LICENSE | 27 + vendor/golang.org/x/text/PATENTS | 22 + .../x/text/transform/examples_test.go | 37 - .../golang.org/x/text/transform/transform.go | 247 +- .../x/text/transform/transform_test.go | 1101 -- .../x/text/unicode/norm/composition_test.go | 130 - .../x/text/unicode/norm/example_iter_test.go | 82 - .../x/text/unicode/norm/forminfo.go | 49 +- .../x/text/unicode/norm/forminfo_test.go | 54 - .../x/text/unicode/norm/iter_test.go | 98 - .../x/text/unicode/norm/maketables.go | 12 +- .../x/text/unicode/norm/norm_test.go | 14 - .../x/text/unicode/norm/normalize.go | 90 +- .../x/text/unicode/norm/normalize_test.go | 1143 -- .../x/text/unicode/norm/readwriter.go | 1 - .../x/text/unicode/norm/readwriter_test.go | 56 - .../golang.org/x/text/unicode/norm/tables.go | 6502 ++++----- .../x/text/unicode/norm/transform_test.go | 101 - .../x/text/unicode/norm/ucd_test.go | 275 - vendor/gopkg.in/fsnotify.v1/README.md | 64 - vendor/gopkg.in/fsnotify.v1/example_test.go | 42 - .../fsnotify.v1/inotify_poller_test.go | 228 - vendor/gopkg.in/fsnotify.v1/inotify_test.go | 343 - .../fsnotify.v1/integration_darwin_test.go | 146 - .../gopkg.in/fsnotify.v1/integration_test.go | 1237 -- .../square/go-jose.v1}/BUG-BOUNTY.md | 0 .../square/go-jose.v1}/CONTRIBUTING.md | 0 .../square/go-jose.v1}/LICENSE | 0 .../square/go-jose.v1}/README.md | 12 +- .../square/go-jose.v1}/asymmetric.go | 26 +- .../square/go-jose.v1}/cipher/cbc_hmac.go | 16 +- .../square/go-jose.v1}/cipher/concat_kdf.go | 2 +- .../square/go-jose.v1}/cipher/ecdh_es.go | 11 + .../square/go-jose.v1}/cipher/key_wrap.go | 0 .../square/go-jose.v1}/crypter.go | 79 +- .../square/go-jose.v1}/doc.go | 2 +- .../square/go-jose.v1}/encoding.go | 5 +- .../gopkg.in/square/go-jose.v1/json/LICENSE | 27 + .../gopkg.in/square/go-jose.v1/json/README.md | 13 + .../gopkg.in/square/go-jose.v1/json/decode.go | 1183 ++ .../gopkg.in/square/go-jose.v1/json/encode.go | 1197 ++ .../gopkg.in/square/go-jose.v1/json/indent.go | 141 + .../square/go-jose.v1/json/scanner.go | 623 + .../gopkg.in/square/go-jose.v1/json/stream.go | 480 + .../gopkg.in/square/go-jose.v1/json/tags.go | 44 + .../square/go-jose.v1}/jwe.go | 3 +- .../square/go-jose.v1}/jwk.go | 100 +- .../square/go-jose.v1}/jws.go | 27 +- .../square/go-jose.v1}/shared.go | 0 .../square/go-jose.v1}/signing.go | 65 +- .../square/go-jose.v1}/symmetric.go | 3 +- .../square/go-jose.v1}/utils.go | 0 vendor/gopkg.in/yaml.v2/LICENSE | 195 +- vendor/gopkg.in/yaml.v2/README.md | 2 +- vendor/gopkg.in/yaml.v2/decode.go | 3 +- vendor/gopkg.in/yaml.v2/decode_test.go | 988 -- vendor/gopkg.in/yaml.v2/emitterc.go | 1 - vendor/gopkg.in/yaml.v2/encode_test.go | 501 - vendor/gopkg.in/yaml.v2/parserc.go | 1 - vendor/gopkg.in/yaml.v2/resolve.go | 11 +- vendor/gopkg.in/yaml.v2/scannerc.go | 4 +- vendor/gopkg.in/yaml.v2/suite_test.go | 12 - vendor/gopkg.in/yaml.v2/yaml.go | 2 +- vendor/manifest | 277 - vendor/vendor.json | 481 + 1748 files changed, 196165 insertions(+), 194500 deletions(-) delete mode 100644 vendor/github.com/BurntSushi/toml/COMPATIBLE delete mode 100644 vendor/github.com/BurntSushi/toml/COPYING delete mode 100644 vendor/github.com/BurntSushi/toml/Makefile delete mode 100644 vendor/github.com/BurntSushi/toml/README.md delete mode 100644 vendor/github.com/BurntSushi/toml/_examples/example.go delete mode 100644 vendor/github.com/BurntSushi/toml/_examples/hard.toml delete mode 100644 vendor/github.com/BurntSushi/toml/_examples/implicit.toml delete mode 100644 vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml delete mode 100644 vendor/github.com/BurntSushi/toml/_examples/readme1.toml delete mode 100644 vendor/github.com/BurntSushi/toml/_examples/readme2.toml delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md delete mode 100644 vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go delete mode 100644 vendor/github.com/BurntSushi/toml/decode.go delete mode 100644 vendor/github.com/BurntSushi/toml/decode_meta.go delete mode 100644 vendor/github.com/BurntSushi/toml/decode_test.go delete mode 100644 vendor/github.com/BurntSushi/toml/doc.go delete mode 100644 vendor/github.com/BurntSushi/toml/encode.go delete mode 100644 vendor/github.com/BurntSushi/toml/encode_test.go delete mode 100644 vendor/github.com/BurntSushi/toml/encoding_types.go delete mode 100644 vendor/github.com/BurntSushi/toml/encoding_types_1.1.go delete mode 100644 vendor/github.com/BurntSushi/toml/lex.go delete mode 100644 vendor/github.com/BurntSushi/toml/parse.go delete mode 100644 vendor/github.com/BurntSushi/toml/session.vim delete mode 100644 vendor/github.com/BurntSushi/toml/type_check.go delete mode 100644 vendor/github.com/BurntSushi/toml/type_fields.go create mode 100644 vendor/github.com/blevesearch/bleve/CONTRIBUTING.md create mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer/custom_analyzer.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzers/detect_lang_analyzer/detect_lang_analyzer.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer/keyword_analyzer.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzers/simple_analyzer/simple_analyzer.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer/standard_analyzer.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/ignore/ignore_byte_array_converter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/json/json_byte_array_converter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/string/string_byte_array_conveter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/char_filters/html_char_filter/html_char_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/char_filters/zero_width_non_joiner/zero_width_non_joiner_char_filter.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional/datetime_optional.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/freq_test.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go rename vendor/github.com/blevesearch/bleve/analysis/{language => lang}/en/stop_words_en.go (98%) delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_filter_ar.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_words_ar.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_filter_bg.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_words_bg.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ca/articles_ca.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_filter_ca.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_words_ca.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_filter_ckb.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_words_ckb.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_filter_cs.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_words_cs.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/da/stemmer_da.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/da/stop_filter_da.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/da/stop_words_da.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/de/stemmer_de.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/de/stop_filter_de.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/de/stop_words_de.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/el/stop_filter_el.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/el/stop_words_el.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/en/stop_filter_en.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/es/stemmer_es.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/es/stop_filter_es.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/es/stop_words_es.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_filter_eu.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_words_eu.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_filter_fa.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_words_fa.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fi/stemmer_fi.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_filter_fi.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_words_fi.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/articles_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/stemmer_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_filter_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_words_fr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ga/articles_ga.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_filter_ga.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_words_ga.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_filter_gl.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_words_gl.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_filter_hi.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_words_hi.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hu/stemmer_hu.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_filter_hu.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_words_hu.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_filter_hy.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_words_hy.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/id/stop_filter_id.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/id/stop_words_id.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/in/scripts.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/articles_it.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/stemmer_it.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/stop_filter_it.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/it/stop_words_it.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/nl/stemmer_nl.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_filter_nl.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_words_nl.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/no/stemmer_no.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/no/stop_filter_no.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/no/stop_words_no.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/porter/stemmer_porter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/pt/stemmer_pt.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_filter_pt.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_words_pt.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ro/stemmer_ro.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_filter_ro.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_words_ro.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ru/stemmer_ru.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_filter_ru.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_words_ru.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/sv/stemmer_sv.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_filter_sv.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_words_sv.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/th/stop_filter_th.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/th/stop_words_th.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/th/unicode_tokenizer_th.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/tr/stemmer_tr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_filter_tr.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_words_tr.go rename vendor/github.com/blevesearch/bleve/analysis/{token_filters/lower_case_filter/lower_case_filter.go => token/lowercase/lowercase.go} (61%) rename vendor/github.com/blevesearch/bleve/analysis/{token_filters => token}/porter/porter.go (50%) rename vendor/github.com/blevesearch/bleve/analysis/{token_filters/stop_tokens_filter/stop_tokens_filter.go => token/stop/stop.go} (56%) delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/README.md delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.cc delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.h delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/compile_cld2.sh delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter/lower_case_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/porter/porter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/README.md delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter/stop_tokens_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_map/custom.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token_map_test.go create mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer.go delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer_test.go rename vendor/github.com/blevesearch/bleve/analysis/{token_map.go => tokenmap.go} (58%) delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/util_test.go create mode 100644 vendor/github.com/blevesearch/bleve/config_app.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_cld2.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_cznicb.go create mode 100644 vendor/github.com/blevesearch/bleve/config_disk.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_forestdb.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_icu.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_kagome.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_leveldb.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_metrics.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_rocksdb.go delete mode 100644 vendor/github.com/blevesearch/bleve/config_stemmer.go delete mode 100644 vendor/github.com/blevesearch/bleve/docs/bleve.png delete mode 100644 vendor/github.com/blevesearch/bleve/docs/build_children.sh delete mode 100644 vendor/github.com/blevesearch/bleve/docs/merge-coverprofile.go delete mode 100644 vendor/github.com/blevesearch/bleve/docs/old_build_script.txt delete mode 100644 vendor/github.com/blevesearch/bleve/docs/project-code-coverage.sh create mode 100644 vendor/github.com/blevesearch/bleve/document/field_boolean.go create mode 100644 vendor/github.com/blevesearch/bleve/document/field_geopoint.go delete mode 100644 vendor/github.com/blevesearch/bleve/document/field_numeric_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/document/indexing_options_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/examples_test.go create mode 100644 vendor/github.com/blevesearch/bleve/geo/README.md create mode 100644 vendor/github.com/blevesearch/bleve/geo/geo.go create mode 100644 vendor/github.com/blevesearch/bleve/geo/geo_dist.go create mode 100644 vendor/github.com/blevesearch/bleve/geo/parse.go create mode 100644 vendor/github.com/blevesearch/bleve/geo/sloppy.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/alias.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/debug.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/doc_count.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/doc_delete.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/doc_get.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/doc_index.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/fields.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/handlers_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/index_create.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/index_delete.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/index_get.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/index_list.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/registry.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/search.go delete mode 100644 vendor/github.com/blevesearch/bleve/http/util.go create mode 100644 vendor/github.com/blevesearch/bleve/index/analysis.go create mode 100644 vendor/github.com/blevesearch/bleve/index/field_cache.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/boltdb/store_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/cznicb/batch.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/cznicb/iterator.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/cznicb/reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/cznicb/writer.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/forestdb/batch.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/forestdb/iterator.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/forestdb/reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/forestdb/store.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/forestdb/store_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/forestdb/writer.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/goleveldb/batch.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/goleveldb/iterator.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/goleveldb/reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/goleveldb/store.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/goleveldb/store_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/goleveldb/util.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/goleveldb/writer.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gorocksdb/batch.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gorocksdb/iterator.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gorocksdb/reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gorocksdb/util.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gorocksdb/writer.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap_test.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/inmem/iterator.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/inmem/reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/inmem/store.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/inmem/store_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/inmem/writer.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/leveldb/batch.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/leveldb/iterator.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/leveldb/reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/leveldb/store.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/leveldb/store_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/leveldb/util.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/leveldb/writer.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/metrics/metrics.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/metrics/metrics_test.go create mode 100644 vendor/github.com/blevesearch/bleve/index/store/multiget.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/null/null.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/store/null/null_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/analysis_pool.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_boltdb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_common_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_cznicb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_forestdb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_goleveldb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gorocksdb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gtreap_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_inmem_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_leveldb_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_null_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/dump.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/dump_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/field_dict.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/field_dict_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/field_index_cache.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/index_reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/reader.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/reader_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/row.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/row_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/stats.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.pb.go delete mode 100644 vendor/github.com/blevesearch/bleve/index/upside_down/upside_down_test.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go rename vendor/github.com/blevesearch/bleve/index/{upside_down => upsidedown}/benchmark_all.sh (100%) mode change 100644 => 100755 create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/row.go rename vendor/github.com/blevesearch/bleve/index/{upside_down => upsidedown}/row_merge.go (68%) create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go create mode 100644 vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go rename vendor/github.com/blevesearch/bleve/index/{upside_down/upside_down.proto => upsidedown/upsidedown.proto} (56%) delete mode 100644 vendor/github.com/blevesearch/bleve/index_alias_impl_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index_meta_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/index_test.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/analysis.go rename vendor/github.com/blevesearch/bleve/{mapping_document.go => mapping/document.go} (52%) create mode 100644 vendor/github.com/blevesearch/bleve/mapping/field.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/index.go create mode 100644 vendor/github.com/blevesearch/bleve/mapping/mapping.go rename vendor/github.com/blevesearch/bleve/{ => mapping}/reflect.go (55%) delete mode 100644 vendor/github.com/blevesearch/bleve/mapping_field.go delete mode 100644 vendor/github.com/blevesearch/bleve/mapping_index.go delete mode 100644 vendor/github.com/blevesearch/bleve/mapping_test.go create mode 100644 vendor/github.com/blevesearch/bleve/numeric/bin.go create mode 100644 vendor/github.com/blevesearch/bleve/numeric/float.go rename vendor/github.com/blevesearch/bleve/{numeric_util => numeric}/prefix_coded.go (61%) delete mode 100644 vendor/github.com/blevesearch/bleve/numeric_util/float.go delete mode 100644 vendor/github.com/blevesearch/bleve/numeric_util/float_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/numeric_util/prefix_coded_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_boolean.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_conjunction.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_date_range.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_disjunction.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_fuzzy.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_match.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_match_all.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_match_none.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_match_phrase.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_numeric_range.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_phrase.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_prefix.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_regexp.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_string.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_string.nex delete mode 100644 vendor/github.com/blevesearch/bleve/query_string.nn.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_string.y delete mode 100644 vendor/github.com/blevesearch/bleve/query_string.y.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_string_parser.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_string_parser_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_term.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/query_wildcard.go delete mode 100644 vendor/github.com/blevesearch/bleve/registry/byte_array_converter.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/cache.go create mode 100644 vendor/github.com/blevesearch/bleve/registry/index_type.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/heap.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/list.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/slice.go create mode 100644 vendor/github.com/blevesearch/bleve/search/collector/topn.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/collectors/search_test.go rename vendor/github.com/blevesearch/bleve/search/{facets => facet}/benchmark_data.txt (100%) create mode 100644 vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go rename vendor/github.com/blevesearch/bleve/search/{facets => facet}/facet_builder_numeric.go (51%) rename vendor/github.com/blevesearch/bleve/search/{facets => facet}/facet_builder_terms.go (52%) delete mode 100644 vendor/github.com/blevesearch/bleve/search/facets/facet_builder_datetime.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/facets/facet_builder_numeric_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/facets/facet_builder_terms_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/facets_builder_test.go rename vendor/github.com/blevesearch/bleve/search/highlight/{fragment_formatters/html/fragment_formatter_html.go => format/html/html.go} (67%) delete mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/ansi/fragment_formatter_ansi.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/html/fragment_formatter_html_test.go rename vendor/github.com/blevesearch/bleve/search/highlight/{fragmenters/simple/fragmenter_simple.go => fragmenter/simple/simple.go} (82%) delete mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/fragmenters/simple/fragmenter_simple_test.go create mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go rename vendor/github.com/blevesearch/bleve/search/highlight/{highlighters => highlighter}/simple/fragment_scorer_simple.go (53%) rename vendor/github.com/blevesearch/bleve/search/highlight/{highlighters => highlighter}/simple/highlighter_simple.go (82%) delete mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/fragment_scorer_simple_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/highlighter_simple_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/highlight/term_locations_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/levenshtein_test.go create mode 100644 vendor/github.com/blevesearch/bleve/search/pool.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/bool_field.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/boolean.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/boost.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/conjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/date_range.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/disjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/docid.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/fuzzy.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/geo_distance.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match_all.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match_none.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/match_phrase.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/numeric_range.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/phrase.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/prefix.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string.y create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string.y.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/regexp.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/term.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/term_range.go create mode 100644 vendor/github.com/blevesearch/bleve/search/query/wildcard.go create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go rename vendor/github.com/blevesearch/bleve/search/{scorers => scorer}/scorer_constant.go (61%) create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go rename vendor/github.com/blevesearch/bleve/search/{scorers => scorer}/scorer_term.go (64%) create mode 100644 vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/scorers/scorer_conjunction.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/scorers/scorer_constant_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/scorers/scorer_disjunction.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/scorers/scorer_term_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/scorers/sqrt_cache.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go rename vendor/github.com/blevesearch/bleve/search/{searchers => searcher}/search_numeric_range.go (53%) create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_term.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go create mode 100644 vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/base_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/ordered_searchers_list.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_boolean.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_boolean_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_fuzzy.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_match_all.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_match_all_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_match_none.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_match_none_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_numeric_range_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_phrase.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_phrase_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_regexp.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_regexp_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_term.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_term_prefix.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/searchers/search_term_test.go create mode 100644 vendor/github.com/blevesearch/bleve/search/sort.go delete mode 100644 vendor/github.com/blevesearch/bleve/search/util_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/search_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/test/integration.go delete mode 100644 vendor/github.com/blevesearch/bleve/test/integration_test.go delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/basic/data/a.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/basic/data/b.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/basic/data/c.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/basic/data/d.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/basic/mapping.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/basic/searches.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/a.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/b.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/c.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/d.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/e.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/f.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/g.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/h.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/i.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/data/j.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/mapping.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/facet/searches.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/fosdem/mapping.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/fosdem/searches.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/phrase/data/a.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/phrase/mapping.json delete mode 100644 vendor/github.com/blevesearch/bleve/test/tests/phrase/searches.json delete mode 100644 vendor/github.com/blevesearch/bleve/utils/bleve_bulkindex/main.go delete mode 100644 vendor/github.com/blevesearch/bleve/utils/bleve_create/main.go delete mode 100644 vendor/github.com/blevesearch/bleve/utils/bleve_dump/main.go delete mode 100644 vendor/github.com/blevesearch/bleve/utils/bleve_index/main.go delete mode 100644 vendor/github.com/blevesearch/bleve/utils/bleve_query/main.go delete mode 100644 vendor/github.com/blevesearch/bleve/utils/bleve_registry/main.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_contains_vowel_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fixes_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fuzz_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_repeat_double_consonant_suffix_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_suffix_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_is_consontant_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_measure_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_string_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_without_lower_casing_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1a_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1b_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1c_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step2_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step3_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step4_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5a_test.go delete mode 100644 vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5b_test.go delete mode 100644 vendor/github.com/blevesearch/segment/export_test.go delete mode 100644 vendor/github.com/blevesearch/segment/ragel/unicode2ragel.rb delete mode 100644 vendor/github.com/blevesearch/segment/ragel/uscript.rl delete mode 100644 vendor/github.com/blevesearch/segment/ragel/uwb.rl delete mode 100644 vendor/github.com/blevesearch/segment/segment_fuzz_test.go delete mode 100644 vendor/github.com/blevesearch/segment/segment_test.go delete mode 100644 vendor/github.com/blevesearch/segment/segment_words_test.go delete mode 100644 vendor/github.com/blevesearch/segment/tables_test.go create mode 100644 vendor/github.com/boltdb/bolt/bolt_ppc.go create mode 100644 vendor/github.com/boltdb/bolt/bolt_ppc64.go delete mode 100644 vendor/github.com/boltdb/bolt/bucket_test.go delete mode 100644 vendor/github.com/boltdb/bolt/cmd/bolt/main.go delete mode 100644 vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go delete mode 100644 vendor/github.com/boltdb/bolt/cursor_test.go delete mode 100644 vendor/github.com/boltdb/bolt/db_test.go delete mode 100644 vendor/github.com/boltdb/bolt/freelist_test.go delete mode 100644 vendor/github.com/boltdb/bolt/node_test.go delete mode 100644 vendor/github.com/boltdb/bolt/page_test.go delete mode 100644 vendor/github.com/boltdb/bolt/quick_test.go delete mode 100644 vendor/github.com/boltdb/bolt/simulation_test.go delete mode 100644 vendor/github.com/boltdb/bolt/tx_test.go delete mode 100644 vendor/github.com/cpuguy83/go-md2man/md2man/md2man.go delete mode 100644 vendor/github.com/cpuguy83/go-md2man/md2man/roff.go delete mode 100644 vendor/github.com/cznic/b/AUTHORS delete mode 100644 vendor/github.com/cznic/b/CONTRIBUTORS delete mode 100644 vendor/github.com/cznic/b/Makefile delete mode 100644 vendor/github.com/cznic/b/README.md delete mode 100644 vendor/github.com/cznic/b/all_test.go delete mode 100644 vendor/github.com/cznic/b/btree.go delete mode 100644 vendor/github.com/cznic/b/doc.go delete mode 100644 vendor/github.com/cznic/b/example/Makefile delete mode 100644 vendor/github.com/cznic/b/example/all_test.go delete mode 100644 vendor/github.com/cznic/b/example/int.go create mode 100644 vendor/github.com/davecgh/go-spew/LICENSE delete mode 100644 vendor/github.com/davecgh/go-spew/spew/common_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/dump_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/dumpnocgo_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/example_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/format_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/internal_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/spew_test.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/testdata/dumpcgo.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md create mode 100644 vendor/github.com/dgrijalva/jwt-go/claims.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/cmd/jwt/app.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/ecdsa_test.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/example_test.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/hmac_test.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/map_claims.go create mode 100644 vendor/github.com/dgrijalva/jwt-go/none.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/parser_test.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/rsa_pss_test.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/rsa_test.go delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/ec256-private.pem delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/ec256-public.pem delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/ec384-private.pem delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/ec384-public.pem delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/ec512-private.pem delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/ec512-public.pem delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/hmacTestKey delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/sample_key delete mode 100644 vendor/github.com/dgrijalva/jwt-go/test/sample_key.pub create mode 100644 vendor/github.com/dsnet/compress/LICENSE.md delete mode 100644 vendor/github.com/dsnet/compress/brotli/brotli_test.go delete mode 100644 vendor/github.com/dsnet/compress/brotli/common_test.go delete mode 100644 vendor/github.com/dsnet/compress/brotli/dict_decoder_test.go delete mode 100644 vendor/github.com/dsnet/compress/brotli/dict_encoder_test.go delete mode 100644 vendor/github.com/dsnet/compress/brotli/prefix_test.go delete mode 100644 vendor/github.com/dsnet/compress/brotli/reader_test.go delete mode 100644 vendor/github.com/dsnet/compress/brotli/transform_test.go delete mode 100644 vendor/github.com/dsnet/compress/brotli/writer_test.go rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/AUTHORS (85%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/CHANGELOG.md (74%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/CONTRIBUTING.md (76%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/LICENSE (100%) create mode 100644 vendor/github.com/fsnotify/fsnotify/README.md create mode 100644 vendor/github.com/fsnotify/fsnotify/fen.go rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/fsnotify.go (72%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/inotify.go (71%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/inotify_poller.go (76%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/kqueue.go (90%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/open_mode_bsd.go (73%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/open_mode_darwin.go (79%) rename vendor/{gopkg.in/fsnotify.v1 => github.com/fsnotify/fsnotify}/windows.go (86%) create mode 100644 vendor/github.com/golang/protobuf/LICENSE delete mode 100644 vendor/github.com/golang/protobuf/proto/all_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/clone_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/equal_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/extensions_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/message_set_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.pb.go delete mode 100644 vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.proto delete mode 100644 vendor/github.com/golang/protobuf/proto/proto3_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/size2_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/size_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/testdata/Makefile delete mode 100644 vendor/github.com/golang/protobuf/proto/testdata/golden_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/testdata/test.pb.go delete mode 100644 vendor/github.com/golang/protobuf/proto/testdata/test.proto delete mode 100644 vendor/github.com/golang/protobuf/proto/text_parser_test.go delete mode 100644 vendor/github.com/golang/protobuf/proto/text_test.go delete mode 100644 vendor/github.com/golang/snappy/AUTHORS delete mode 100644 vendor/github.com/golang/snappy/CONTRIBUTORS delete mode 100644 vendor/github.com/golang/snappy/README delete mode 100644 vendor/github.com/golang/snappy/cmd/snappytool/main.cpp delete mode 100644 vendor/github.com/golang/snappy/decode.go delete mode 100644 vendor/github.com/golang/snappy/decode_amd64.go delete mode 100644 vendor/github.com/golang/snappy/decode_amd64.s delete mode 100644 vendor/github.com/golang/snappy/decode_other.go delete mode 100644 vendor/github.com/golang/snappy/encode.go delete mode 100644 vendor/github.com/golang/snappy/snappy.go delete mode 100644 vendor/github.com/golang/snappy/snappy_test.go delete mode 100644 vendor/github.com/golang/snappy/testdata/pi.txt delete mode 100644 vendor/github.com/golang/snappy/testdata/pi.txt.rawsnappy delete mode 100644 vendor/github.com/gorilla/websocket/bench_test.go create mode 100644 vendor/github.com/gorilla/websocket/client_clone.go create mode 100644 vendor/github.com/gorilla/websocket/client_clone_legacy.go delete mode 100644 vendor/github.com/gorilla/websocket/client_server_test.go delete mode 100644 vendor/github.com/gorilla/websocket/client_test.go create mode 100644 vendor/github.com/gorilla/websocket/compression.go create mode 100644 vendor/github.com/gorilla/websocket/conn_read.go create mode 100644 vendor/github.com/gorilla/websocket/conn_read_legacy.go delete mode 100644 vendor/github.com/gorilla/websocket/conn_test.go delete mode 100644 vendor/github.com/gorilla/websocket/example_test.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/autobahn/README.md delete mode 100644 vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json delete mode 100644 vendor/github.com/gorilla/websocket/examples/autobahn/server.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/chat/README.md delete mode 100644 vendor/github.com/gorilla/websocket/examples/chat/conn.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/chat/home.html delete mode 100644 vendor/github.com/gorilla/websocket/examples/chat/hub.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/chat/main.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/command/README.md delete mode 100644 vendor/github.com/gorilla/websocket/examples/command/home.html delete mode 100644 vendor/github.com/gorilla/websocket/examples/command/main.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/echo/README.md delete mode 100644 vendor/github.com/gorilla/websocket/examples/echo/client.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/echo/server.go delete mode 100644 vendor/github.com/gorilla/websocket/examples/filewatch/README.md delete mode 100644 vendor/github.com/gorilla/websocket/examples/filewatch/main.go delete mode 100644 vendor/github.com/gorilla/websocket/json_test.go create mode 100644 vendor/github.com/gorilla/websocket/mask.go create mode 100644 vendor/github.com/gorilla/websocket/mask_safe.go create mode 100644 vendor/github.com/gorilla/websocket/prepared.go delete mode 100644 vendor/github.com/gorilla/websocket/server_test.go delete mode 100644 vendor/github.com/gorilla/websocket/util_test.go create mode 100644 vendor/github.com/hashicorp/hcl/appveyor.yml delete mode 100644 vendor/github.com/hashicorp/hcl/decoder_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/ast/ast_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/dir.ignore delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/file.ignore delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/good.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/error_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/parser_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/array_comment_2.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_colon.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/assign_deep.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_lastline.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/comment_single.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/complex_key.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/empty.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/list_comma.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/missing_braces.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/multiple.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/old.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_basic.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/structure_empty.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/types.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/parser/test-fixtures/unterminated_object.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/nodes.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/printer.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/printer_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.input delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/list.golden delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/printer/testdata/list.input delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/scanner/scanner_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/strconv/quote_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/array_comment.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_colon.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_deep.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment_single.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex_key.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/empty.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list_comma.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/multiple.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/old.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_basic.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_empty.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/test-fixtures/types.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/hcl/token/token_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/hcl_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/json/parser/parser_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/json/parser/test-fixtures/array.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/parser/test-fixtures/basic.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/parser/test-fixtures/object.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/parser/test-fixtures/types.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/scanner/scanner_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/json/test-fixtures/array.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/test-fixtures/basic.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/test-fixtures/object.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/test-fixtures/types.json delete mode 100644 vendor/github.com/hashicorp/hcl/json/token/token_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/lex_test.go delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/basic.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/basic.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/basic_int_string.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/basic_squish.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/empty.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/escape.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/flat.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/float.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/float.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/interpolate_escape.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/multiline.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/multiline.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/multiline_bad.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/multiline_no_eof.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/multiline_no_marker.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/nested_block_comment.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/object_list.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/scientific.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/scientific.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/slice_expand.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure2.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure2.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure_flat.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure_flatmap.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure_list.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure_list.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure_list_deep.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure_multi.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/structure_multi.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/terraform_heroku.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/terraform_heroku.json delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/tfvars.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/unterminated_block_comment.hcl delete mode 100644 vendor/github.com/hashicorp/hcl/test-fixtures/unterminated_brace.hcl create mode 100644 vendor/github.com/inconshreveable/mousetrap/LICENSE create mode 100644 vendor/github.com/inconshreveable/mousetrap/README.md create mode 100644 vendor/github.com/inconshreveable/mousetrap/trap_others.go create mode 100644 vendor/github.com/inconshreveable/mousetrap/trap_windows.go create mode 100644 vendor/github.com/inconshreveable/mousetrap/trap_windows_1.4.go rename vendor/github.com/{shurcooL/sanitized_anchor_name => jpillora/backoff}/LICENSE (86%) delete mode 100644 vendor/github.com/jpillora/backoff/backoff_test.go delete mode 100644 vendor/github.com/kr/pretty/Readme delete mode 100644 vendor/github.com/kr/pretty/diff.go delete mode 100644 vendor/github.com/kr/pretty/diff_test.go delete mode 100644 vendor/github.com/kr/pretty/example_test.go delete mode 100644 vendor/github.com/kr/pretty/formatter.go delete mode 100644 vendor/github.com/kr/pretty/formatter_test.go delete mode 100644 vendor/github.com/kr/pretty/pretty.go delete mode 100644 vendor/github.com/kr/pretty/zero.go delete mode 100644 vendor/github.com/kr/text/Readme delete mode 100644 vendor/github.com/kr/text/colwriter/Readme delete mode 100644 vendor/github.com/kr/text/colwriter/column.go delete mode 100644 vendor/github.com/kr/text/colwriter/column_test.go delete mode 100644 vendor/github.com/kr/text/doc.go delete mode 100644 vendor/github.com/kr/text/indent.go delete mode 100644 vendor/github.com/kr/text/indent_test.go delete mode 100644 vendor/github.com/kr/text/mc/Readme delete mode 100644 vendor/github.com/kr/text/mc/mc.go delete mode 100644 vendor/github.com/kr/text/wrap.go delete mode 100644 vendor/github.com/kr/text/wrap_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/LICENSE delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/README.md delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/TODO delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/bootstrap_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/export_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/fixture_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/foundation_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer_test.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run.go delete mode 100644 vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run_test.go delete mode 100644 vendor/github.com/magiconair/properties/benchmark_test.go delete mode 100644 vendor/github.com/magiconair/properties/decode_test.go delete mode 100644 vendor/github.com/magiconair/properties/example_test.go delete mode 100644 vendor/github.com/magiconair/properties/integrate_test.go delete mode 100644 vendor/github.com/magiconair/properties/load_test.go delete mode 100644 vendor/github.com/magiconair/properties/properties_test.go rename vendor/github.com/{kr/pretty/License => matryer/resync/LICENSE} (89%) delete mode 100644 vendor/github.com/matryer/resync/once_test.go create mode 100644 vendor/github.com/miekg/dns/AUTHORS create mode 100644 vendor/github.com/miekg/dns/CONTRIBUTORS create mode 100644 vendor/github.com/miekg/dns/COPYRIGHT create mode 100644 vendor/github.com/miekg/dns/LICENSE create mode 100644 vendor/github.com/miekg/dns/README.md create mode 100644 vendor/github.com/miekg/dns/client.go create mode 100644 vendor/github.com/miekg/dns/clientconfig.go create mode 100644 vendor/github.com/miekg/dns/compress_generate.go create mode 100644 vendor/github.com/miekg/dns/dane.go create mode 100644 vendor/github.com/miekg/dns/defaults.go create mode 100644 vendor/github.com/miekg/dns/dns.go create mode 100644 vendor/github.com/miekg/dns/dnssec.go create mode 100644 vendor/github.com/miekg/dns/dnssec_keygen.go create mode 100644 vendor/github.com/miekg/dns/dnssec_keyscan.go create mode 100644 vendor/github.com/miekg/dns/dnssec_privkey.go create mode 100644 vendor/github.com/miekg/dns/doc.go create mode 100644 vendor/github.com/miekg/dns/edns.go create mode 100644 vendor/github.com/miekg/dns/format.go create mode 100644 vendor/github.com/miekg/dns/generate.go create mode 100644 vendor/github.com/miekg/dns/labels.go create mode 100644 vendor/github.com/miekg/dns/msg.go create mode 100644 vendor/github.com/miekg/dns/msg_generate.go create mode 100644 vendor/github.com/miekg/dns/msg_helpers.go create mode 100644 vendor/github.com/miekg/dns/nsecx.go create mode 100644 vendor/github.com/miekg/dns/privaterr.go create mode 100644 vendor/github.com/miekg/dns/rawmsg.go create mode 100644 vendor/github.com/miekg/dns/reverse.go create mode 100644 vendor/github.com/miekg/dns/sanitize.go create mode 100644 vendor/github.com/miekg/dns/scan.go create mode 100644 vendor/github.com/miekg/dns/scan_rr.go create mode 100644 vendor/github.com/miekg/dns/scanner.go create mode 100644 vendor/github.com/miekg/dns/server.go create mode 100644 vendor/github.com/miekg/dns/sig0.go create mode 100644 vendor/github.com/miekg/dns/singleinflight.go create mode 100644 vendor/github.com/miekg/dns/smimea.go create mode 100644 vendor/github.com/miekg/dns/tlsa.go create mode 100644 vendor/github.com/miekg/dns/tsig.go create mode 100644 vendor/github.com/miekg/dns/types.go create mode 100644 vendor/github.com/miekg/dns/types_generate.go create mode 100644 vendor/github.com/miekg/dns/udp.go create mode 100644 vendor/github.com/miekg/dns/udp_linux.go create mode 100644 vendor/github.com/miekg/dns/udp_other.go create mode 100644 vendor/github.com/miekg/dns/udp_windows.go create mode 100644 vendor/github.com/miekg/dns/update.go create mode 100644 vendor/github.com/miekg/dns/xfr.go create mode 100644 vendor/github.com/miekg/dns/zcompress.go create mode 100644 vendor/github.com/miekg/dns/zmsg.go create mode 100644 vendor/github.com/miekg/dns/ztypes.go delete mode 100644 vendor/github.com/mitchellh/go-homedir/homedir_test.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/decode_hooks_test.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/mapstructure_benchmark_test.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/mapstructure_bugs_test.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/mapstructure_examples_test.go delete mode 100644 vendor/github.com/mitchellh/mapstructure/mapstructure_test.go create mode 100644 vendor/github.com/pelletier/go-buffruneio/README.md create mode 100644 vendor/github.com/pelletier/go-buffruneio/buffruneio.go create mode 100644 vendor/github.com/pelletier/go-toml/LICENSE create mode 100644 vendor/github.com/pelletier/go-toml/README.md create mode 100755 vendor/github.com/pelletier/go-toml/clean.sh create mode 100644 vendor/github.com/pelletier/go-toml/doc.go rename vendor/github.com/{BurntSushi/toml/_examples/invalid.toml => pelletier/go-toml/example-crlf.toml} (71%) rename vendor/github.com/{BurntSushi/toml/_examples => pelletier/go-toml}/example.toml (89%) create mode 100644 vendor/github.com/pelletier/go-toml/keysparsing.go create mode 100644 vendor/github.com/pelletier/go-toml/lexer.go create mode 100644 vendor/github.com/pelletier/go-toml/marshal.go create mode 100644 vendor/github.com/pelletier/go-toml/marshal_test.toml create mode 100644 vendor/github.com/pelletier/go-toml/match.go create mode 100644 vendor/github.com/pelletier/go-toml/parser.go create mode 100644 vendor/github.com/pelletier/go-toml/position.go create mode 100644 vendor/github.com/pelletier/go-toml/query.go create mode 100644 vendor/github.com/pelletier/go-toml/querylexer.go create mode 100644 vendor/github.com/pelletier/go-toml/queryparser.go create mode 100755 vendor/github.com/pelletier/go-toml/test.sh create mode 100644 vendor/github.com/pelletier/go-toml/token.go create mode 100644 vendor/github.com/pelletier/go-toml/toml.go create mode 100644 vendor/github.com/pelletier/go-toml/tomltree_create.go create mode 100644 vendor/github.com/pelletier/go-toml/tomltree_write.go create mode 100644 vendor/github.com/pmezard/go-difflib/LICENSE delete mode 100644 vendor/github.com/pmezard/go-difflib/difflib/difflib_test.go delete mode 100644 vendor/github.com/russross/blackfriday/LICENSE.txt delete mode 100644 vendor/github.com/russross/blackfriday/README.md delete mode 100644 vendor/github.com/russross/blackfriday/block.go delete mode 100644 vendor/github.com/russross/blackfriday/block_test.go delete mode 100644 vendor/github.com/russross/blackfriday/html.go delete mode 100644 vendor/github.com/russross/blackfriday/inline.go delete mode 100644 vendor/github.com/russross/blackfriday/inline_test.go delete mode 100644 vendor/github.com/russross/blackfriday/latex.go delete mode 100644 vendor/github.com/russross/blackfriday/markdown.go delete mode 100644 vendor/github.com/russross/blackfriday/ref_test.go delete mode 100644 vendor/github.com/russross/blackfriday/smartypants.go delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Amps and angle encoding.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Amps and angle encoding.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Auto links.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Auto links.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Backslash escapes.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Backslash escapes.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Blockquotes with code blocks.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Blockquotes with code blocks.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Blocks.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Blocks.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Spans.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Code Spans.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines no empty line before block.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines no empty line before block.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Hard-wrapped paragraphs with list-like lines.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Horizontal rules.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Horizontal rules.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Advanced).html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Advanced).text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Simple).html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML (Simple).text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML comments.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Inline HTML comments.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, inline style.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, inline style.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, reference style.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, reference style.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, shortcut references.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Links, shortcut references.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Literal quotes in titles.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Literal quotes in titles.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Basics.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Basics.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Syntax.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Markdown Documentation - Syntax.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Nested blockquotes.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Nested blockquotes.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Ordered and unordered lists.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Ordered and unordered lists.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Strong and em together.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Strong and em together.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Tabs.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Tabs.text delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Tidyness.html delete mode 100644 vendor/github.com/russross/blackfriday/testdata/Tidyness.text delete mode 100644 vendor/github.com/ryszard/goskiplist/skiplist/skiplist.go delete mode 100644 vendor/github.com/ryszard/goskiplist/skiplist/skiplist_test.go delete mode 100644 vendor/github.com/shurcooL/sanitized_anchor_name/README.md delete mode 100644 vendor/github.com/shurcooL/sanitized_anchor_name/main.go delete mode 100644 vendor/github.com/shurcooL/sanitized_anchor_name/main_test.go create mode 100644 vendor/github.com/spf13/afero/LICENSE.txt create mode 100644 vendor/github.com/spf13/afero/README.md create mode 100644 vendor/github.com/spf13/afero/afero.go create mode 100644 vendor/github.com/spf13/afero/appveyor.yml create mode 100644 vendor/github.com/spf13/afero/basepath.go create mode 100644 vendor/github.com/spf13/afero/cacheOnReadFs.go create mode 100644 vendor/github.com/spf13/afero/const_bsds.go create mode 100644 vendor/github.com/spf13/afero/const_win_unix.go create mode 100644 vendor/github.com/spf13/afero/copyOnWriteFs.go create mode 100644 vendor/github.com/spf13/afero/httpFs.go create mode 100644 vendor/github.com/spf13/afero/ioutil.go create mode 100644 vendor/github.com/spf13/afero/mem/dir.go create mode 100644 vendor/github.com/spf13/afero/mem/dirmap.go create mode 100644 vendor/github.com/spf13/afero/mem/file.go create mode 100644 vendor/github.com/spf13/afero/memmap.go rename vendor/github.com/spf13/{cobra/cobra/main.go => afero/memradix.go} (79%) create mode 100644 vendor/github.com/spf13/afero/os.go create mode 100644 vendor/github.com/spf13/afero/path.go create mode 100644 vendor/github.com/spf13/afero/readonlyfs.go create mode 100644 vendor/github.com/spf13/afero/regexpfs.go create mode 100644 vendor/github.com/spf13/afero/unionFile.go create mode 100644 vendor/github.com/spf13/afero/util.go create mode 100644 vendor/github.com/spf13/cast/Makefile delete mode 100644 vendor/github.com/spf13/cast/cast_test.go delete mode 100644 vendor/github.com/spf13/cobra/bash_completions_test.go delete mode 100644 vendor/github.com/spf13/cobra/cobra/cmd/add.go delete mode 100644 vendor/github.com/spf13/cobra/cobra/cmd/helpers.go delete mode 100644 vendor/github.com/spf13/cobra/cobra/cmd/helpers_test.go delete mode 100644 vendor/github.com/spf13/cobra/cobra/cmd/init.go delete mode 100644 vendor/github.com/spf13/cobra/cobra/cmd/licenses.go delete mode 100644 vendor/github.com/spf13/cobra/cobra/cmd/root.go delete mode 100644 vendor/github.com/spf13/cobra/cobra_test.go delete mode 100644 vendor/github.com/spf13/cobra/command_test.go delete mode 100644 vendor/github.com/spf13/cobra/doc/cmd_test.go delete mode 100644 vendor/github.com/spf13/cobra/doc/man_docs.go delete mode 100644 vendor/github.com/spf13/cobra/doc/man_docs.md delete mode 100644 vendor/github.com/spf13/cobra/doc/man_docs_test.go delete mode 100644 vendor/github.com/spf13/cobra/doc/man_examples_test.go delete mode 100644 vendor/github.com/spf13/cobra/doc/md_docs.go delete mode 100644 vendor/github.com/spf13/cobra/doc/md_docs.md delete mode 100644 vendor/github.com/spf13/cobra/doc/md_docs_test.go delete mode 100644 vendor/github.com/spf13/cobra/doc/util.go create mode 100644 vendor/github.com/spf13/jwalterweatherman/default_notepad.go delete mode 100644 vendor/github.com/spf13/jwalterweatherman/jww_test.go create mode 100644 vendor/github.com/spf13/jwalterweatherman/log_counter.go create mode 100644 vendor/github.com/spf13/jwalterweatherman/notepad.go delete mode 100644 vendor/github.com/spf13/jwalterweatherman/thatswhyyoualwaysleaveanote.go create mode 100644 vendor/github.com/spf13/pflag/bool_slice.go delete mode 100644 vendor/github.com/spf13/pflag/bool_test.go delete mode 100644 vendor/github.com/spf13/pflag/count_test.go delete mode 100644 vendor/github.com/spf13/pflag/example_test.go delete mode 100644 vendor/github.com/spf13/pflag/export_test.go delete mode 100644 vendor/github.com/spf13/pflag/flag_test.go delete mode 100644 vendor/github.com/spf13/pflag/golangflag_test.go delete mode 100644 vendor/github.com/spf13/pflag/int_slice_test.go create mode 100644 vendor/github.com/spf13/pflag/ip_slice.go delete mode 100644 vendor/github.com/spf13/pflag/ip_test.go delete mode 100644 vendor/github.com/spf13/pflag/ipnet_test.go create mode 100644 vendor/github.com/spf13/pflag/string_array.go delete mode 100644 vendor/github.com/spf13/pflag/string_slice_test.go create mode 100644 vendor/github.com/spf13/pflag/uint_slice.go delete mode 100644 vendor/github.com/spf13/pflag/verify/all.sh delete mode 100644 vendor/github.com/spf13/pflag/verify/gofmt.sh delete mode 100644 vendor/github.com/spf13/pflag/verify/golint.sh delete mode 100644 vendor/github.com/spf13/viper/flags_test.go create mode 100644 vendor/github.com/spf13/viper/nohup.out delete mode 100644 vendor/github.com/spf13/viper/remote/remote.go delete mode 100644 vendor/github.com/spf13/viper/viper_test.go delete mode 100644 vendor/github.com/square/go-jose/asymmetric_test.go delete mode 100644 vendor/github.com/square/go-jose/cipher/cbc_hmac_test.go delete mode 100644 vendor/github.com/square/go-jose/cipher/concat_kdf_test.go delete mode 100644 vendor/github.com/square/go-jose/cipher/ecdh_es_test.go delete mode 100644 vendor/github.com/square/go-jose/cipher/key_wrap_test.go delete mode 100644 vendor/github.com/square/go-jose/crypter_test.go delete mode 100644 vendor/github.com/square/go-jose/doc_test.go delete mode 100644 vendor/github.com/square/go-jose/encoding_test.go delete mode 100644 vendor/github.com/square/go-jose/jose-util/README.md delete mode 100644 vendor/github.com/square/go-jose/jose-util/jose-util.t delete mode 100644 vendor/github.com/square/go-jose/jose-util/main.go delete mode 100644 vendor/github.com/square/go-jose/jwe_test.go delete mode 100644 vendor/github.com/square/go-jose/jwk_test.go delete mode 100644 vendor/github.com/square/go-jose/jws_test.go delete mode 100644 vendor/github.com/square/go-jose/signing_test.go delete mode 100644 vendor/github.com/square/go-jose/symmetric_test.go delete mode 100644 vendor/github.com/square/go-jose/utils_test.go delete mode 100644 vendor/github.com/steveyen/gtreap/treap_test.go create mode 100644 vendor/github.com/stretchr/testify/LICENSE delete mode 100644 vendor/github.com/stretchr/testify/assert/assertions_test.go delete mode 100644 vendor/github.com/stretchr/testify/assert/forward_assertions_test.go delete mode 100644 vendor/github.com/stretchr/testify/assert/http_assertions_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/batch.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/batch_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/bench2_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/bench_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/lru.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/comparer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/corrupt_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_state.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_write.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/doc.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/errors.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/errors/errors.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/external_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter/filter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/journal/journal.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/key.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/key_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/bench_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/options.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_compaction.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_record.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_record_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/block_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/table.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/table_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/writer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/db.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/kv.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/buffer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/crc32.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/hash.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/pool.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/pool_legacy.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/range.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/version.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/version_test.go delete mode 100644 vendor/github.com/willf/bitset/Makefile delete mode 100644 vendor/github.com/willf/bitset/README.md delete mode 100644 vendor/github.com/willf/bitset/RELEASE delete mode 100644 vendor/github.com/willf/bitset/VERSION delete mode 100644 vendor/github.com/willf/bitset/bitset.go delete mode 100644 vendor/github.com/willf/bitset/bitset_test.go delete mode 100644 vendor/github.com/willf/bitset/popcnt.go delete mode 100644 vendor/github.com/willf/bitset/popcnt_amd64.go delete mode 100644 vendor/github.com/willf/bitset/popcnt_amd64.s delete mode 100644 vendor/github.com/willf/bitset/popcnt_generic.go delete mode 100644 vendor/github.com/willf/bitset/popcnt_test.go rename vendor/github.com/{kr/text/License => xenolf/lego/LICENSE} (86%) create mode 100644 vendor/github.com/xenolf/lego/acme/challenges.go delete mode 100644 vendor/github.com/xenolf/lego/acme/client_test.go delete mode 100644 vendor/github.com/xenolf/lego/acme/crypto_test.go create mode 100644 vendor/github.com/xenolf/lego/acme/dns_challenge_manual.go create mode 100644 vendor/github.com/xenolf/lego/acme/http_challenge_server.go delete mode 100644 vendor/github.com/xenolf/lego/acme/http_challenge_test.go delete mode 100644 vendor/github.com/xenolf/lego/acme/http_test.go create mode 100644 vendor/github.com/xenolf/lego/acme/provider.go create mode 100644 vendor/github.com/xenolf/lego/acme/tls_sni_challenge_server.go delete mode 100644 vendor/github.com/xenolf/lego/acme/tls_sni_challenge_test.go create mode 100644 vendor/github.com/xenolf/lego/acme/utils.go rename vendor/{github.com/willf/bitset => golang.org/x/crypto}/LICENSE (96%) create mode 100644 vendor/golang.org/x/crypto/PATENTS delete mode 100644 vendor/golang.org/x/crypto/ocsp/ocsp_test.go delete mode 100644 vendor/golang.org/x/crypto/sha3/doc.go delete mode 100644 vendor/golang.org/x/crypto/sha3/hashes.go delete mode 100644 vendor/golang.org/x/crypto/sha3/keccakf.go delete mode 100644 vendor/golang.org/x/crypto/sha3/register.go delete mode 100644 vendor/golang.org/x/crypto/sha3/sha3.go delete mode 100644 vendor/golang.org/x/crypto/sha3/sha3_test.go delete mode 100644 vendor/golang.org/x/crypto/sha3/shake.go delete mode 100644 vendor/golang.org/x/crypto/sha3/testdata/keccakKats.json.deflate delete mode 100644 vendor/golang.org/x/crypto/sha3/xor.go delete mode 100644 vendor/golang.org/x/crypto/sha3/xor_generic.go delete mode 100644 vendor/golang.org/x/crypto/sha3/xor_unaligned.go delete mode 100644 vendor/golang.org/x/crypto/ssh/terminal/terminal.go delete mode 100644 vendor/golang.org/x/crypto/ssh/terminal/terminal_test.go delete mode 100644 vendor/golang.org/x/crypto/ssh/terminal/util.go delete mode 100644 vendor/golang.org/x/crypto/ssh/terminal/util_bsd.go delete mode 100644 vendor/golang.org/x/crypto/ssh/terminal/util_linux.go delete mode 100644 vendor/golang.org/x/crypto/ssh/terminal/util_windows.go rename vendor/{github.com/golang/snappy => golang.org/x/net}/LICENSE (95%) create mode 100644 vendor/golang.org/x/net/PATENTS create mode 100644 vendor/golang.org/x/net/context/context.go create mode 100644 vendor/golang.org/x/net/context/go17.go create mode 100644 vendor/golang.org/x/net/context/pre_go17.go create mode 100644 vendor/golang.org/x/net/publicsuffix/gen.go create mode 100644 vendor/golang.org/x/net/publicsuffix/list.go create mode 100644 vendor/golang.org/x/net/publicsuffix/table.go rename vendor/{github.com/cznic/b => golang.org/x/sys}/LICENSE (92%) create mode 100644 vendor/golang.org/x/sys/PATENTS create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_darwin_arm64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_dragonfly_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_freebsd_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_freebsd_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_freebsd_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_arm64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_mips64x.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_mipsx.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_ppc64x.s create mode 100644 vendor/golang.org/x/sys/unix/asm_linux_s390x.s create mode 100644 vendor/golang.org/x/sys/unix/asm_netbsd_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_netbsd_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_netbsd_arm.s create mode 100644 vendor/golang.org/x/sys/unix/asm_openbsd_386.s create mode 100644 vendor/golang.org/x/sys/unix/asm_openbsd_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/asm_solaris_amd64.s create mode 100644 vendor/golang.org/x/sys/unix/bluetooth_linux.go create mode 100644 vendor/golang.org/x/sys/unix/constants.go create mode 100644 vendor/golang.org/x/sys/unix/dirent.go create mode 100644 vendor/golang.org/x/sys/unix/endian_big.go create mode 100644 vendor/golang.org/x/sys/unix/endian_little.go create mode 100644 vendor/golang.org/x/sys/unix/env_unix.go create mode 100644 vendor/golang.org/x/sys/unix/env_unset.go create mode 100644 vendor/golang.org/x/sys/unix/flock.go create mode 100644 vendor/golang.org/x/sys/unix/flock_linux_32bit.go create mode 100644 vendor/golang.org/x/sys/unix/gccgo.go create mode 100644 vendor/golang.org/x/sys/unix/gccgo_c.c create mode 100644 vendor/golang.org/x/sys/unix/gccgo_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/gccgo_linux_sparc64.go create mode 100755 vendor/golang.org/x/sys/unix/mkall.sh create mode 100755 vendor/golang.org/x/sys/unix/mkerrors.sh create mode 100644 vendor/golang.org/x/sys/unix/mkpost.go create mode 100755 vendor/golang.org/x/sys/unix/mksyscall.pl create mode 100755 vendor/golang.org/x/sys/unix/mksyscall_solaris.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysctl_openbsd.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_darwin.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_dragonfly.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_freebsd.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_linux.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_netbsd.pl create mode 100755 vendor/golang.org/x/sys/unix/mksysnum_openbsd.pl create mode 100644 vendor/golang.org/x/sys/unix/openbsd_pledge.go create mode 100644 vendor/golang.org/x/sys/unix/race.go create mode 100644 vendor/golang.org/x/sys/unix/race0.go create mode 100644 vendor/golang.org/x/sys/unix/sockcmsg_linux.go create mode 100644 vendor/golang.org/x/sys/unix/sockcmsg_unix.go create mode 100644 vendor/golang.org/x/sys/unix/str.go create mode 100644 vendor/golang.org/x/sys/unix/syscall.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_bsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_dragonfly.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_amd64_gc.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_mipsx.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_ppc64x.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_no_getwd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_solaris.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_unix.go create mode 100644 vendor/golang.org/x/sys/unix/syscall_unix_gc.go create mode 100644 vendor/golang.org/x/sys/unix/types_darwin.go create mode 100644 vendor/golang.org/x/sys/unix/types_dragonfly.go create mode 100644 vendor/golang.org/x/sys/unix/types_freebsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_linux.go create mode 100644 vendor/golang.org/x/sys/unix/types_netbsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/types_solaris.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zerrors_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysctl_openbsd.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/zsysnum_solaris_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_dragonfly_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mips.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_netbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_netbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_netbsd_arm.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_openbsd_386.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_openbsd_amd64.go create mode 100644 vendor/golang.org/x/sys/unix/ztypes_solaris_amd64.go create mode 100644 vendor/golang.org/x/text/LICENSE create mode 100644 vendor/golang.org/x/text/PATENTS delete mode 100644 vendor/golang.org/x/text/transform/examples_test.go delete mode 100644 vendor/golang.org/x/text/transform/transform_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/composition_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/example_iter_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/forminfo_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/iter_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/norm_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/normalize_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/readwriter_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/transform_test.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/ucd_test.go delete mode 100644 vendor/gopkg.in/fsnotify.v1/README.md delete mode 100644 vendor/gopkg.in/fsnotify.v1/example_test.go delete mode 100644 vendor/gopkg.in/fsnotify.v1/inotify_poller_test.go delete mode 100644 vendor/gopkg.in/fsnotify.v1/inotify_test.go delete mode 100644 vendor/gopkg.in/fsnotify.v1/integration_darwin_test.go delete mode 100644 vendor/gopkg.in/fsnotify.v1/integration_test.go rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/BUG-BOUNTY.md (100%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/CONTRIBUTING.md (100%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/LICENSE (100%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/README.md (88%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/asymmetric.go (95%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/cipher/cbc_hmac.go (90%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/cipher/concat_kdf.go (92%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/cipher/ecdh_es.go (79%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/cipher/key_wrap.go (100%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/crypter.go (81%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/doc.go (94%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/encoding.go (97%) create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/LICENSE create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/README.md create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/decode.go create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/encode.go create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/indent.go create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/scanner.go create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/stream.go create mode 100644 vendor/gopkg.in/square/go-jose.v1/json/tags.go rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/jwe.go (99%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/jwk.go (80%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/jws.go (88%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/shared.go (100%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/signing.go (74%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/symmetric.go (99%) rename vendor/{github.com/square/go-jose => gopkg.in/square/go-jose.v1}/utils.go (100%) delete mode 100644 vendor/gopkg.in/yaml.v2/decode_test.go delete mode 100644 vendor/gopkg.in/yaml.v2/encode_test.go delete mode 100644 vendor/gopkg.in/yaml.v2/suite_test.go delete mode 100644 vendor/manifest create mode 100644 vendor/vendor.json diff --git a/letsencrypt/letsencrypt.go b/letsencrypt/letsencrypt.go index 4bf0aeb4..e0d2fd78 100644 --- a/letsencrypt/letsencrypt.go +++ b/letsencrypt/letsencrypt.go @@ -24,8 +24,8 @@ func Run(dir, domain, email, port string) (*state, error) { return nil, err } - client, err := acme.NewClient(URL, &user, KeySize) - client.ExcludeChallenges([]string{"tls-sni-01"}) + client, err := acme.NewClient(URL, &user, acme.RSA2048) + client.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01}) client.SetHTTPAddress(port) if user.Registration == nil { @@ -123,7 +123,7 @@ func (s *state) setOCSP(ocsp []byte) { } func (s *state) obtain() error { - cert, errors := s.client.ObtainCertificate([]string{s.domain}, true, nil) + cert, errors := s.client.ObtainCertificate([]string{s.domain}, true, nil, false) if err := errors[s.domain]; err != nil { if _, ok := err.(acme.TOSError); ok { err := s.client.AgreeToTOS() @@ -180,7 +180,7 @@ func (s *state) renew() bool { meta.PrivateKey = key Renew: - newMeta, err := s.client.RenewCertificate(meta, true) + newMeta, err := s.client.RenewCertificate(meta, true, false) if err != nil { if _, ok := err.(acme.TOSError); ok { err := s.client.AgreeToTOS() diff --git a/letsencrypt/user.go b/letsencrypt/user.go index c6a905aa..43c940fc 100644 --- a/letsencrypt/user.go +++ b/letsencrypt/user.go @@ -1,6 +1,7 @@ package letsencrypt import ( + "crypto" "crypto/rand" "crypto/rsa" "crypto/x509" @@ -17,7 +18,7 @@ const defaultUser = "default" type User struct { Email string Registration *acme.RegistrationResource - key *rsa.PrivateKey + key crypto.PrivateKey } func (u User) GetEmail() string { @@ -28,7 +29,7 @@ func (u User) GetRegistration() *acme.RegistrationResource { return u.Registration } -func (u User) GetPrivateKey() *rsa.PrivateKey { +func (u User) GetPrivateKey() crypto.PrivateKey { return u.key } @@ -86,7 +87,7 @@ func saveUser(user User) error { return ioutil.WriteFile(directory.UserRegistration(user.Email), jsonBytes, 0600) } -func loadRSAPrivateKey(file string) (*rsa.PrivateKey, error) { +func loadRSAPrivateKey(file string) (crypto.PrivateKey, error) { keyBytes, err := ioutil.ReadFile(file) if err != nil { return nil, err @@ -95,8 +96,10 @@ func loadRSAPrivateKey(file string) (*rsa.PrivateKey, error) { return x509.ParsePKCS1PrivateKey(keyBlock.Bytes) } -func saveRSAPrivateKey(key *rsa.PrivateKey, file string) error { - pemKey := pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)} +func saveRSAPrivateKey(key crypto.PrivateKey, file string) error { + pemKey := pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key.(*rsa.PrivateKey)), + } keyOut, err := os.Create(file) if err != nil { return err diff --git a/letsencrypt/user_test.go b/letsencrypt/user_test.go index 7a73e563..078c2a62 100644 --- a/letsencrypt/user_test.go +++ b/letsencrypt/user_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/xenolf/lego/acme" ) func tempdir() string { @@ -14,18 +13,10 @@ func tempdir() string { } func testUser(t *testing.T, email string) { - reg := &acme.RegistrationResource{ - URI: "test.com", - Body: acme.Registration{ - Agreement: "agree?", - }, - } - user, err := newUser(email) assert.Nil(t, err) key := user.GetPrivateKey() assert.NotNil(t, key) - user.Registration = reg err = saveUser(user) assert.Nil(t, err) @@ -34,7 +25,6 @@ func testUser(t *testing.T, email string) { assert.Nil(t, err) assert.Equal(t, email, user.GetEmail()) assert.Equal(t, key, user.GetPrivateKey()) - assert.Equal(t, reg, user.GetRegistration()) } func TestUser(t *testing.T) { diff --git a/server/auth.go b/server/auth.go index bd3d5031..ae6751b0 100644 --- a/server/auth.go +++ b/server/auth.go @@ -55,7 +55,8 @@ func handleAuth(w http.ResponseWriter, r *http.Request) *Session { token, err := parseToken(cookie.Value) if err == nil && token.Valid { - userID := uint64(token.Claims["UserID"].(float64)) + claims := token.Claims.(jwt.MapClaims) + userID := uint64(claims["UserID"].(float64)) log.Println(r.RemoteAddr, "[Auth] GET", r.URL.Path, "| Valid token | User ID:", userID) @@ -91,7 +92,8 @@ func newUser(w http.ResponseWriter, r *http.Request) *Session { go session.run() token := jwt.New(jwt.SigningMethodHS256) - token.Claims["UserID"] = user.ID + claims := token.Claims.(jwt.MapClaims) + claims["UserID"] = user.ID tokenString, err := token.SignedString(hmacKey) if err != nil { return nil diff --git a/storage/user_messages.go b/storage/user_messages.go index d62908b4..dfaa992b 100644 --- a/storage/user_messages.go +++ b/storage/user_messages.go @@ -118,7 +118,8 @@ func (u *User) SearchMessages(server, channel, q string) ([]Message, error) { contentQuery.SetField("content") contentQuery.SetFuzziness(2) - query := bleve.NewBooleanQuery([]bleve.Query{serverQuery, channelQuery, contentQuery}, nil, nil) + query := bleve.NewBooleanQuery() + query.AddMust(serverQuery, channelQuery, contentQuery) search := bleve.NewSearchRequest(query) searchResults, err := u.messageIndex.Search(search) diff --git a/vendor/github.com/BurntSushi/toml/COMPATIBLE b/vendor/github.com/BurntSushi/toml/COMPATIBLE deleted file mode 100644 index 21e0938c..00000000 --- a/vendor/github.com/BurntSushi/toml/COMPATIBLE +++ /dev/null @@ -1,3 +0,0 @@ -Compatible with TOML version -[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) - diff --git a/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/BurntSushi/toml/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/BurntSushi/toml/Makefile b/vendor/github.com/BurntSushi/toml/Makefile deleted file mode 100644 index 3600848d..00000000 --- a/vendor/github.com/BurntSushi/toml/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -install: - go install ./... - -test: install - go test -v - toml-test toml-test-decoder - toml-test -encoder toml-test-encoder - -fmt: - gofmt -w *.go */*.go - colcheck *.go */*.go - -tags: - find ./ -name '*.go' -print0 | xargs -0 gotags > TAGS - -push: - git push origin master - git push github master - diff --git a/vendor/github.com/BurntSushi/toml/README.md b/vendor/github.com/BurntSushi/toml/README.md deleted file mode 100644 index 5a5df637..00000000 --- a/vendor/github.com/BurntSushi/toml/README.md +++ /dev/null @@ -1,220 +0,0 @@ -## TOML parser and encoder for Go with reflection - -TOML stands for Tom's Obvious, Minimal Language. This Go package provides a -reflection interface similar to Go's standard library `json` and `xml` -packages. This package also supports the `encoding.TextUnmarshaler` and -`encoding.TextMarshaler` interfaces so that you can define custom data -representations. (There is an example of this below.) - -Spec: https://github.com/mojombo/toml - -Compatible with TOML version -[v0.2.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.2.0.md) - -Documentation: http://godoc.org/github.com/BurntSushi/toml - -Installation: - -```bash -go get github.com/BurntSushi/toml -``` - -Try the toml validator: - -```bash -go get github.com/BurntSushi/toml/cmd/tomlv -tomlv some-toml-file.toml -``` - -[![Build status](https://api.travis-ci.org/BurntSushi/toml.png)](https://travis-ci.org/BurntSushi/toml) - - -### Testing - -This package passes all tests in -[toml-test](https://github.com/BurntSushi/toml-test) for both the decoder -and the encoder. - -### Examples - -This package works similarly to how the Go standard library handles `XML` -and `JSON`. Namely, data is loaded into Go values via reflection. - -For the simplest example, consider some TOML file as just a list of keys -and values: - -```toml -Age = 25 -Cats = [ "Cauchy", "Plato" ] -Pi = 3.14 -Perfection = [ 6, 28, 496, 8128 ] -DOB = 1987-07-05T05:45:00Z -``` - -Which could be defined in Go as: - -```go -type Config struct { - Age int - Cats []string - Pi float64 - Perfection []int - DOB time.Time // requires `import time` -} -``` - -And then decoded with: - -```go -var conf Config -if _, err := toml.Decode(tomlData, &conf); err != nil { - // handle error -} -``` - -You can also use struct tags if your struct field name doesn't map to a TOML -key value directly: - -```toml -some_key_NAME = "wat" -``` - -```go -type TOML struct { - ObscureKey string `toml:"some_key_NAME"` -} -``` - -### Using the `encoding.TextUnmarshaler` interface - -Here's an example that automatically parses duration strings into -`time.Duration` values: - -```toml -[[song]] -name = "Thunder Road" -duration = "4m49s" - -[[song]] -name = "Stairway to Heaven" -duration = "8m03s" -``` - -Which can be decoded with: - -```go -type song struct { - Name string - Duration duration -} -type songs struct { - Song []song -} -var favorites songs -if _, err := toml.Decode(blob, &favorites); err != nil { - log.Fatal(err) -} - -for _, s := range favorites.Song { - fmt.Printf("%s (%s)\n", s.Name, s.Duration) -} -``` - -And you'll also need a `duration` type that satisfies the -`encoding.TextUnmarshaler` interface: - -```go -type duration struct { - time.Duration -} - -func (d *duration) UnmarshalText(text []byte) error { - var err error - d.Duration, err = time.ParseDuration(string(text)) - return err -} -``` - -### More complex usage - -Here's an example of how to load the example from the official spec page: - -```toml -# This is a TOML document. Boom. - -title = "TOML Example" - -[owner] -name = "Tom Preston-Werner" -organization = "GitHub" -bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." -dob = 1979-05-27T07:32:00Z # First class dates? Why not? - -[database] -server = "192.168.1.1" -ports = [ 8001, 8001, 8002 ] -connection_max = 5000 -enabled = true - -[servers] - - # You can indent as you please. Tabs or spaces. TOML don't care. - [servers.alpha] - ip = "10.0.0.1" - dc = "eqdc10" - - [servers.beta] - ip = "10.0.0.2" - dc = "eqdc10" - -[clients] -data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it - -# Line breaks are OK when inside arrays -hosts = [ - "alpha", - "omega" -] -``` - -And the corresponding Go types are: - -```go -type tomlConfig struct { - Title string - Owner ownerInfo - DB database `toml:"database"` - Servers map[string]server - Clients clients -} - -type ownerInfo struct { - Name string - Org string `toml:"organization"` - Bio string - DOB time.Time -} - -type database struct { - Server string - Ports []int - ConnMax int `toml:"connection_max"` - Enabled bool -} - -type server struct { - IP string - DC string -} - -type clients struct { - Data [][]interface{} - Hosts []string -} -``` - -Note that a case insensitive match will be tried if an exact match can't be -found. - -A working example of the above can be found in `_examples/example.{go,toml}`. - diff --git a/vendor/github.com/BurntSushi/toml/_examples/example.go b/vendor/github.com/BurntSushi/toml/_examples/example.go deleted file mode 100644 index 79f31f27..00000000 --- a/vendor/github.com/BurntSushi/toml/_examples/example.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "fmt" - "time" - - "github.com/BurntSushi/toml" -) - -type tomlConfig struct { - Title string - Owner ownerInfo - DB database `toml:"database"` - Servers map[string]server - Clients clients -} - -type ownerInfo struct { - Name string - Org string `toml:"organization"` - Bio string - DOB time.Time -} - -type database struct { - Server string - Ports []int - ConnMax int `toml:"connection_max"` - Enabled bool -} - -type server struct { - IP string - DC string -} - -type clients struct { - Data [][]interface{} - Hosts []string -} - -func main() { - var config tomlConfig - if _, err := toml.DecodeFile("example.toml", &config); err != nil { - fmt.Println(err) - return - } - - fmt.Printf("Title: %s\n", config.Title) - fmt.Printf("Owner: %s (%s, %s), Born: %s\n", - config.Owner.Name, config.Owner.Org, config.Owner.Bio, - config.Owner.DOB) - fmt.Printf("Database: %s %v (Max conn. %d), Enabled? %v\n", - config.DB.Server, config.DB.Ports, config.DB.ConnMax, - config.DB.Enabled) - for serverName, server := range config.Servers { - fmt.Printf("Server: %s (%s, %s)\n", serverName, server.IP, server.DC) - } - fmt.Printf("Client data: %v\n", config.Clients.Data) - fmt.Printf("Client hosts: %v\n", config.Clients.Hosts) -} diff --git a/vendor/github.com/BurntSushi/toml/_examples/hard.toml b/vendor/github.com/BurntSushi/toml/_examples/hard.toml deleted file mode 100644 index 26145d2b..00000000 --- a/vendor/github.com/BurntSushi/toml/_examples/hard.toml +++ /dev/null @@ -1,22 +0,0 @@ -# Test file for TOML -# Only this one tries to emulate a TOML file written by a user of the kind of parser writers probably hate -# This part you'll really hate - -[the] -test_string = "You'll hate me after this - #" # " Annoying, isn't it? - - [the.hard] - test_array = [ "] ", " # "] # ] There you go, parse this! - test_array2 = [ "Test #11 ]proved that", "Experiment #9 was a success" ] - # You didn't think it'd as easy as chucking out the last #, did you? - another_test_string = " Same thing, but with a string #" - harder_test_string = " And when \"'s are in the string, along with # \"" # "and comments are there too" - # Things will get harder - - [the.hard.bit#] - what? = "You don't think some user won't do that?" - multi_line_array = [ - "]", - # ] Oh yes I did - ] - diff --git a/vendor/github.com/BurntSushi/toml/_examples/implicit.toml b/vendor/github.com/BurntSushi/toml/_examples/implicit.toml deleted file mode 100644 index 1dea5ceb..00000000 --- a/vendor/github.com/BurntSushi/toml/_examples/implicit.toml +++ /dev/null @@ -1,4 +0,0 @@ -# [x] you -# [x.y] don't -# [x.y.z] need these -[x.y.z.w] # for this to work diff --git a/vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml b/vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml deleted file mode 100644 index 74e9e337..00000000 --- a/vendor/github.com/BurntSushi/toml/_examples/invalid-apples.toml +++ /dev/null @@ -1,6 +0,0 @@ -# DO NOT WANT -[fruit] -type = "apple" - -[fruit.type] -apple = "yes" diff --git a/vendor/github.com/BurntSushi/toml/_examples/readme1.toml b/vendor/github.com/BurntSushi/toml/_examples/readme1.toml deleted file mode 100644 index 3e1261d4..00000000 --- a/vendor/github.com/BurntSushi/toml/_examples/readme1.toml +++ /dev/null @@ -1,5 +0,0 @@ -Age = 25 -Cats = [ "Cauchy", "Plato" ] -Pi = 3.14 -Perfection = [ 6, 28, 496, 8128 ] -DOB = 1987-07-05T05:45:00Z diff --git a/vendor/github.com/BurntSushi/toml/_examples/readme2.toml b/vendor/github.com/BurntSushi/toml/_examples/readme2.toml deleted file mode 100644 index b51cd934..00000000 --- a/vendor/github.com/BurntSushi/toml/_examples/readme2.toml +++ /dev/null @@ -1 +0,0 @@ -some_key_NAME = "wat" diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md deleted file mode 100644 index 24421eb7..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Implements the TOML test suite interface - -This is an implementation of the interface expected by -[toml-test](https://github.com/BurntSushi/toml-test) for my -[toml parser written in Go](https://github.com/BurntSushi/toml). -In particular, it maps TOML data on `stdin` to a JSON format on `stdout`. - - -Compatible with TOML version -[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) - -Compatible with `toml-test` version -[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) - diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go deleted file mode 100644 index 14e75570..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/main.go +++ /dev/null @@ -1,90 +0,0 @@ -// Command toml-test-decoder satisfies the toml-test interface for testing -// TOML decoders. Namely, it accepts TOML on stdin and outputs JSON on stdout. -package main - -import ( - "encoding/json" - "flag" - "fmt" - "log" - "os" - "path" - "time" - - "github.com/BurntSushi/toml" -) - -func init() { - log.SetFlags(0) - - flag.Usage = usage - flag.Parse() -} - -func usage() { - log.Printf("Usage: %s < toml-file\n", path.Base(os.Args[0])) - flag.PrintDefaults() - - os.Exit(1) -} - -func main() { - if flag.NArg() != 0 { - flag.Usage() - } - - var tmp interface{} - if _, err := toml.DecodeReader(os.Stdin, &tmp); err != nil { - log.Fatalf("Error decoding TOML: %s", err) - } - - typedTmp := translate(tmp) - if err := json.NewEncoder(os.Stdout).Encode(typedTmp); err != nil { - log.Fatalf("Error encoding JSON: %s", err) - } -} - -func translate(tomlData interface{}) interface{} { - switch orig := tomlData.(type) { - case map[string]interface{}: - typed := make(map[string]interface{}, len(orig)) - for k, v := range orig { - typed[k] = translate(v) - } - return typed - case []map[string]interface{}: - typed := make([]map[string]interface{}, len(orig)) - for i, v := range orig { - typed[i] = translate(v).(map[string]interface{}) - } - return typed - case []interface{}: - typed := make([]interface{}, len(orig)) - for i, v := range orig { - typed[i] = translate(v) - } - - // We don't really need to tag arrays, but let's be future proof. - // (If TOML ever supports tuples, we'll need this.) - return tag("array", typed) - case time.Time: - return tag("datetime", orig.Format("2006-01-02T15:04:05Z")) - case bool: - return tag("bool", fmt.Sprintf("%v", orig)) - case int64: - return tag("integer", fmt.Sprintf("%d", orig)) - case float64: - return tag("float", fmt.Sprintf("%v", orig)) - case string: - return tag("string", orig) - } - - panic(fmt.Sprintf("Unknown type: %T", tomlData)) -} - -func tag(typeName string, data interface{}) map[string]interface{} { - return map[string]interface{}{ - "type": typeName, - "value": data, - } -} diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md deleted file mode 100644 index 45a603f2..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Implements the TOML test suite interface for TOML encoders - -This is an implementation of the interface expected by -[toml-test](https://github.com/BurntSushi/toml-test) for the -[TOML encoder](https://github.com/BurntSushi/toml). -In particular, it maps JSON data on `stdin` to a TOML format on `stdout`. - - -Compatible with TOML version -[v0.2.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.2.0.md) - -Compatible with `toml-test` version -[v0.2.0](https://github.com/BurntSushi/toml-test/tree/v0.2.0) - diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go deleted file mode 100644 index 092cc684..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/main.go +++ /dev/null @@ -1,131 +0,0 @@ -// Command toml-test-encoder satisfies the toml-test interface for testing -// TOML encoders. Namely, it accepts JSON on stdin and outputs TOML on stdout. -package main - -import ( - "encoding/json" - "flag" - "log" - "os" - "path" - "strconv" - "time" - - "github.com/BurntSushi/toml" -) - -func init() { - log.SetFlags(0) - - flag.Usage = usage - flag.Parse() -} - -func usage() { - log.Printf("Usage: %s < json-file\n", path.Base(os.Args[0])) - flag.PrintDefaults() - - os.Exit(1) -} - -func main() { - if flag.NArg() != 0 { - flag.Usage() - } - - var tmp interface{} - if err := json.NewDecoder(os.Stdin).Decode(&tmp); err != nil { - log.Fatalf("Error decoding JSON: %s", err) - } - - tomlData := translate(tmp) - if err := toml.NewEncoder(os.Stdout).Encode(tomlData); err != nil { - log.Fatalf("Error encoding TOML: %s", err) - } -} - -func translate(typedJson interface{}) interface{} { - switch v := typedJson.(type) { - case map[string]interface{}: - if len(v) == 2 && in("type", v) && in("value", v) { - return untag(v) - } - m := make(map[string]interface{}, len(v)) - for k, v2 := range v { - m[k] = translate(v2) - } - return m - case []interface{}: - tabArray := make([]map[string]interface{}, len(v)) - for i := range v { - if m, ok := translate(v[i]).(map[string]interface{}); ok { - tabArray[i] = m - } else { - log.Fatalf("JSON arrays may only contain objects. This " + - "corresponds to only tables being allowed in " + - "TOML table arrays.") - } - } - return tabArray - } - log.Fatalf("Unrecognized JSON format '%T'.", typedJson) - panic("unreachable") -} - -func untag(typed map[string]interface{}) interface{} { - t := typed["type"].(string) - v := typed["value"] - switch t { - case "string": - return v.(string) - case "integer": - v := v.(string) - n, err := strconv.Atoi(v) - if err != nil { - log.Fatalf("Could not parse '%s' as integer: %s", v, err) - } - return n - case "float": - v := v.(string) - f, err := strconv.ParseFloat(v, 64) - if err != nil { - log.Fatalf("Could not parse '%s' as float64: %s", v, err) - } - return f - case "datetime": - v := v.(string) - t, err := time.Parse("2006-01-02T15:04:05Z", v) - if err != nil { - log.Fatalf("Could not parse '%s' as a datetime: %s", v, err) - } - return t - case "bool": - v := v.(string) - switch v { - case "true": - return true - case "false": - return false - } - log.Fatalf("Could not parse '%s' as a boolean.", v) - case "array": - v := v.([]interface{}) - array := make([]interface{}, len(v)) - for i := range v { - if m, ok := v[i].(map[string]interface{}); ok { - array[i] = untag(m) - } else { - log.Fatalf("Arrays may only contain other arrays or "+ - "primitive values, but found a '%T'.", m) - } - } - return array - } - log.Fatalf("Unrecognized tag type '%s'.", t) - panic("unreachable") -} - -func in(key string, m map[string]interface{}) bool { - _, ok := m[key] - return ok -} diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING b/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING deleted file mode 100644 index 5a8e3325..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING +++ /dev/null @@ -1,14 +0,0 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 - - Copyright (C) 2004 Sam Hocevar - - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. - - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. - diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md b/vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md deleted file mode 100644 index 5df0dc32..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/tomlv/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# TOML Validator - -If Go is installed, it's simple to try it out: - -```bash -go get github.com/BurntSushi/toml/cmd/tomlv -tomlv some-toml-file.toml -``` - -You can see the types of every key in a TOML file with: - -```bash -tomlv -types some-toml-file.toml -``` - -At the moment, only one error message is reported at a time. Error messages -include line numbers. No output means that the files given are valid TOML, or -there is a bug in `tomlv`. - -Compatible with TOML version -[v0.1.0](https://github.com/mojombo/toml/blob/master/versions/toml-v0.1.0.md) - diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go b/vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go deleted file mode 100644 index c7d689a7..00000000 --- a/vendor/github.com/BurntSushi/toml/cmd/tomlv/main.go +++ /dev/null @@ -1,61 +0,0 @@ -// Command tomlv validates TOML documents and prints each key's type. -package main - -import ( - "flag" - "fmt" - "log" - "os" - "path" - "strings" - "text/tabwriter" - - "github.com/BurntSushi/toml" -) - -var ( - flagTypes = false -) - -func init() { - log.SetFlags(0) - - flag.BoolVar(&flagTypes, "types", flagTypes, - "When set, the types of every defined key will be shown.") - - flag.Usage = usage - flag.Parse() -} - -func usage() { - log.Printf("Usage: %s toml-file [ toml-file ... ]\n", - path.Base(os.Args[0])) - flag.PrintDefaults() - - os.Exit(1) -} - -func main() { - if flag.NArg() < 1 { - flag.Usage() - } - for _, f := range flag.Args() { - var tmp interface{} - md, err := toml.DecodeFile(f, &tmp) - if err != nil { - log.Fatalf("Error in '%s': %s", f, err) - } - if flagTypes { - printTypes(md) - } - } -} - -func printTypes(md toml.MetaData) { - tabw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - for _, key := range md.Keys() { - fmt.Fprintf(tabw, "%s%s\t%s\n", - strings.Repeat(" ", len(key)-1), key, md.Type(key...)) - } - tabw.Flush() -} diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go deleted file mode 100644 index 58d1c26f..00000000 --- a/vendor/github.com/BurntSushi/toml/decode.go +++ /dev/null @@ -1,493 +0,0 @@ -package toml - -import ( - "fmt" - "io" - "io/ioutil" - "math" - "reflect" - "strings" - "time" -) - -var e = fmt.Errorf - -// Unmarshaler is the interface implemented by objects that can unmarshal a -// TOML description of themselves. -type Unmarshaler interface { - UnmarshalTOML(interface{}) error -} - -// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`. -func Unmarshal(p []byte, v interface{}) error { - _, err := Decode(string(p), v) - return err -} - -// Primitive is a TOML value that hasn't been decoded into a Go value. -// When using the various `Decode*` functions, the type `Primitive` may -// be given to any value, and its decoding will be delayed. -// -// A `Primitive` value can be decoded using the `PrimitiveDecode` function. -// -// The underlying representation of a `Primitive` value is subject to change. -// Do not rely on it. -// -// N.B. Primitive values are still parsed, so using them will only avoid -// the overhead of reflection. They can be useful when you don't know the -// exact type of TOML data until run time. -type Primitive struct { - undecoded interface{} - context Key -} - -// DEPRECATED! -// -// Use MetaData.PrimitiveDecode instead. -func PrimitiveDecode(primValue Primitive, v interface{}) error { - md := MetaData{decoded: make(map[string]bool)} - return md.unify(primValue.undecoded, rvalue(v)) -} - -// PrimitiveDecode is just like the other `Decode*` functions, except it -// decodes a TOML value that has already been parsed. Valid primitive values -// can *only* be obtained from values filled by the decoder functions, -// including this method. (i.e., `v` may contain more `Primitive` -// values.) -// -// Meta data for primitive values is included in the meta data returned by -// the `Decode*` functions with one exception: keys returned by the Undecoded -// method will only reflect keys that were decoded. Namely, any keys hidden -// behind a Primitive will be considered undecoded. Executing this method will -// update the undecoded keys in the meta data. (See the example.) -func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { - md.context = primValue.context - defer func() { md.context = nil }() - return md.unify(primValue.undecoded, rvalue(v)) -} - -// Decode will decode the contents of `data` in TOML format into a pointer -// `v`. -// -// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be -// used interchangeably.) -// -// TOML arrays of tables correspond to either a slice of structs or a slice -// of maps. -// -// TOML datetimes correspond to Go `time.Time` values. -// -// All other TOML types (float, string, int, bool and array) correspond -// to the obvious Go types. -// -// An exception to the above rules is if a type implements the -// encoding.TextUnmarshaler interface. In this case, any primitive TOML value -// (floats, strings, integers, booleans and datetimes) will be converted to -// a byte string and given to the value's UnmarshalText method. See the -// Unmarshaler example for a demonstration with time duration strings. -// -// Key mapping -// -// TOML keys can map to either keys in a Go map or field names in a Go -// struct. The special `toml` struct tag may be used to map TOML keys to -// struct fields that don't match the key name exactly. (See the example.) -// A case insensitive match to struct names will be tried if an exact match -// can't be found. -// -// The mapping between TOML values and Go values is loose. That is, there -// may exist TOML values that cannot be placed into your representation, and -// there may be parts of your representation that do not correspond to -// TOML values. This loose mapping can be made stricter by using the IsDefined -// and/or Undecoded methods on the MetaData returned. -// -// This decoder will not handle cyclic types. If a cyclic type is passed, -// `Decode` will not terminate. -func Decode(data string, v interface{}) (MetaData, error) { - p, err := parse(data) - if err != nil { - return MetaData{}, err - } - md := MetaData{ - p.mapping, p.types, p.ordered, - make(map[string]bool, len(p.ordered)), nil, - } - return md, md.unify(p.mapping, rvalue(v)) -} - -// DecodeFile is just like Decode, except it will automatically read the -// contents of the file at `fpath` and decode it for you. -func DecodeFile(fpath string, v interface{}) (MetaData, error) { - bs, err := ioutil.ReadFile(fpath) - if err != nil { - return MetaData{}, err - } - return Decode(string(bs), v) -} - -// DecodeReader is just like Decode, except it will consume all bytes -// from the reader and decode it for you. -func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { - bs, err := ioutil.ReadAll(r) - if err != nil { - return MetaData{}, err - } - return Decode(string(bs), v) -} - -// unify performs a sort of type unification based on the structure of `rv`, -// which is the client representation. -// -// Any type mismatch produces an error. Finding a type that we don't know -// how to handle produces an unsupported type error. -func (md *MetaData) unify(data interface{}, rv reflect.Value) error { - - // Special case. Look for a `Primitive` value. - if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() { - // Save the undecoded data and the key context into the primitive - // value. - context := make(Key, len(md.context)) - copy(context, md.context) - rv.Set(reflect.ValueOf(Primitive{ - undecoded: data, - context: context, - })) - return nil - } - - // Special case. Unmarshaler Interface support. - if rv.CanAddr() { - if v, ok := rv.Addr().Interface().(Unmarshaler); ok { - return v.UnmarshalTOML(data) - } - } - - // Special case. Handle time.Time values specifically. - // TODO: Remove this code when we decide to drop support for Go 1.1. - // This isn't necessary in Go 1.2 because time.Time satisfies the encoding - // interfaces. - if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) { - return md.unifyDatetime(data, rv) - } - - // Special case. Look for a value satisfying the TextUnmarshaler interface. - if v, ok := rv.Interface().(TextUnmarshaler); ok { - return md.unifyText(data, v) - } - // BUG(burntsushi) - // The behavior here is incorrect whenever a Go type satisfies the - // encoding.TextUnmarshaler interface but also corresponds to a TOML - // hash or array. In particular, the unmarshaler should only be applied - // to primitive TOML values. But at this point, it will be applied to - // all kinds of values and produce an incorrect error whenever those values - // are hashes or arrays (including arrays of tables). - - k := rv.Kind() - - // laziness - if k >= reflect.Int && k <= reflect.Uint64 { - return md.unifyInt(data, rv) - } - switch k { - case reflect.Ptr: - elem := reflect.New(rv.Type().Elem()) - err := md.unify(data, reflect.Indirect(elem)) - if err != nil { - return err - } - rv.Set(elem) - return nil - case reflect.Struct: - return md.unifyStruct(data, rv) - case reflect.Map: - return md.unifyMap(data, rv) - case reflect.Array: - return md.unifyArray(data, rv) - case reflect.Slice: - return md.unifySlice(data, rv) - case reflect.String: - return md.unifyString(data, rv) - case reflect.Bool: - return md.unifyBool(data, rv) - case reflect.Interface: - // we only support empty interfaces. - if rv.NumMethod() > 0 { - return e("Unsupported type '%s'.", rv.Kind()) - } - return md.unifyAnything(data, rv) - case reflect.Float32: - fallthrough - case reflect.Float64: - return md.unifyFloat64(data, rv) - } - return e("Unsupported type '%s'.", rv.Kind()) -} - -func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { - tmap, ok := mapping.(map[string]interface{}) - if !ok { - return mismatch(rv, "map", mapping) - } - - for key, datum := range tmap { - var f *field - fields := cachedTypeFields(rv.Type()) - for i := range fields { - ff := &fields[i] - if ff.name == key { - f = ff - break - } - if f == nil && strings.EqualFold(ff.name, key) { - f = ff - } - } - if f != nil { - subv := rv - for _, i := range f.index { - subv = indirect(subv.Field(i)) - } - if isUnifiable(subv) { - md.decoded[md.context.add(key).String()] = true - md.context = append(md.context, key) - if err := md.unify(datum, subv); err != nil { - return e("Type mismatch for '%s.%s': %s", - rv.Type().String(), f.name, err) - } - md.context = md.context[0 : len(md.context)-1] - } else if f.name != "" { - // Bad user! No soup for you! - return e("Field '%s.%s' is unexported, and therefore cannot "+ - "be loaded with reflection.", rv.Type().String(), f.name) - } - } - } - return nil -} - -func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error { - tmap, ok := mapping.(map[string]interface{}) - if !ok { - return badtype("map", mapping) - } - if rv.IsNil() { - rv.Set(reflect.MakeMap(rv.Type())) - } - for k, v := range tmap { - md.decoded[md.context.add(k).String()] = true - md.context = append(md.context, k) - - rvkey := indirect(reflect.New(rv.Type().Key())) - rvval := reflect.Indirect(reflect.New(rv.Type().Elem())) - if err := md.unify(v, rvval); err != nil { - return err - } - md.context = md.context[0 : len(md.context)-1] - - rvkey.SetString(k) - rv.SetMapIndex(rvkey, rvval) - } - return nil -} - -func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error { - datav := reflect.ValueOf(data) - if datav.Kind() != reflect.Slice { - return badtype("slice", data) - } - sliceLen := datav.Len() - if sliceLen != rv.Len() { - return e("expected array length %d; got TOML array of length %d", - rv.Len(), sliceLen) - } - return md.unifySliceArray(datav, rv) -} - -func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error { - datav := reflect.ValueOf(data) - if datav.Kind() != reflect.Slice { - return badtype("slice", data) - } - sliceLen := datav.Len() - if rv.IsNil() || rv.Len() < datav.Len() { - rv.Set(reflect.MakeSlice(rv.Type(), sliceLen, sliceLen)) - } - rv.SetLen(datav.Len()) - return md.unifySliceArray(datav, rv) -} - -func (md *MetaData) unifySliceArray(data, rv reflect.Value) error { - sliceLen := data.Len() - for i := 0; i < sliceLen; i++ { - v := data.Index(i).Interface() - sliceval := indirect(rv.Index(i)) - if err := md.unify(v, sliceval); err != nil { - return err - } - } - return nil -} - -func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error { - if _, ok := data.(time.Time); ok { - rv.Set(reflect.ValueOf(data)) - return nil - } - return badtype("time.Time", data) -} - -func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error { - if s, ok := data.(string); ok { - rv.SetString(s) - return nil - } - return badtype("string", data) -} - -func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error { - if num, ok := data.(float64); ok { - switch rv.Kind() { - case reflect.Float32: - fallthrough - case reflect.Float64: - rv.SetFloat(num) - default: - panic("bug") - } - return nil - } - return badtype("float", data) -} - -func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { - if num, ok := data.(int64); ok { - if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 { - switch rv.Kind() { - case reflect.Int, reflect.Int64: - // No bounds checking necessary. - case reflect.Int8: - if num < math.MinInt8 || num > math.MaxInt8 { - return e("Value '%d' is out of range for int8.", num) - } - case reflect.Int16: - if num < math.MinInt16 || num > math.MaxInt16 { - return e("Value '%d' is out of range for int16.", num) - } - case reflect.Int32: - if num < math.MinInt32 || num > math.MaxInt32 { - return e("Value '%d' is out of range for int32.", num) - } - } - rv.SetInt(num) - } else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 { - unum := uint64(num) - switch rv.Kind() { - case reflect.Uint, reflect.Uint64: - // No bounds checking necessary. - case reflect.Uint8: - if num < 0 || unum > math.MaxUint8 { - return e("Value '%d' is out of range for uint8.", num) - } - case reflect.Uint16: - if num < 0 || unum > math.MaxUint16 { - return e("Value '%d' is out of range for uint16.", num) - } - case reflect.Uint32: - if num < 0 || unum > math.MaxUint32 { - return e("Value '%d' is out of range for uint32.", num) - } - } - rv.SetUint(unum) - } else { - panic("unreachable") - } - return nil - } - return badtype("integer", data) -} - -func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error { - if b, ok := data.(bool); ok { - rv.SetBool(b) - return nil - } - return badtype("boolean", data) -} - -func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error { - rv.Set(reflect.ValueOf(data)) - return nil -} - -func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error { - var s string - switch sdata := data.(type) { - case TextMarshaler: - text, err := sdata.MarshalText() - if err != nil { - return err - } - s = string(text) - case fmt.Stringer: - s = sdata.String() - case string: - s = sdata - case bool: - s = fmt.Sprintf("%v", sdata) - case int64: - s = fmt.Sprintf("%d", sdata) - case float64: - s = fmt.Sprintf("%f", sdata) - default: - return badtype("primitive (string-like)", data) - } - if err := v.UnmarshalText([]byte(s)); err != nil { - return err - } - return nil -} - -// rvalue returns a reflect.Value of `v`. All pointers are resolved. -func rvalue(v interface{}) reflect.Value { - return indirect(reflect.ValueOf(v)) -} - -// indirect returns the value pointed to by a pointer. -// Pointers are followed until the value is not a pointer. -// New values are allocated for each nil pointer. -// -// An exception to this rule is if the value satisfies an interface of -// interest to us (like encoding.TextUnmarshaler). -func indirect(v reflect.Value) reflect.Value { - if v.Kind() != reflect.Ptr { - if v.CanAddr() { - pv := v.Addr() - if _, ok := pv.Interface().(TextUnmarshaler); ok { - return pv - } - } - return v - } - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - return indirect(reflect.Indirect(v)) -} - -func isUnifiable(rv reflect.Value) bool { - if rv.CanSet() { - return true - } - if _, ok := rv.Interface().(TextUnmarshaler); ok { - return true - } - return false -} - -func badtype(expected string, data interface{}) error { - return e("Expected %s but found '%T'.", expected, data) -} - -func mismatch(user reflect.Value, expected string, data interface{}) error { - return e("Type mismatch for %s. Expected %s but found '%T'.", - user.Type().String(), expected, data) -} diff --git a/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/BurntSushi/toml/decode_meta.go deleted file mode 100644 index ef6f545f..00000000 --- a/vendor/github.com/BurntSushi/toml/decode_meta.go +++ /dev/null @@ -1,122 +0,0 @@ -package toml - -import "strings" - -// MetaData allows access to meta information about TOML data that may not -// be inferrable via reflection. In particular, whether a key has been defined -// and the TOML type of a key. -type MetaData struct { - mapping map[string]interface{} - types map[string]tomlType - keys []Key - decoded map[string]bool - context Key // Used only during decoding. -} - -// IsDefined returns true if the key given exists in the TOML data. The key -// should be specified hierarchially. e.g., -// -// // access the TOML key 'a.b.c' -// IsDefined("a", "b", "c") -// -// IsDefined will return false if an empty key given. Keys are case sensitive. -func (md *MetaData) IsDefined(key ...string) bool { - if len(key) == 0 { - return false - } - - var hash map[string]interface{} - var ok bool - var hashOrVal interface{} = md.mapping - for _, k := range key { - if hash, ok = hashOrVal.(map[string]interface{}); !ok { - return false - } - if hashOrVal, ok = hash[k]; !ok { - return false - } - } - return true -} - -// Type returns a string representation of the type of the key specified. -// -// Type will return the empty string if given an empty key or a key that -// does not exist. Keys are case sensitive. -func (md *MetaData) Type(key ...string) string { - fullkey := strings.Join(key, ".") - if typ, ok := md.types[fullkey]; ok { - return typ.typeString() - } - return "" -} - -// Key is the type of any TOML key, including key groups. Use (MetaData).Keys -// to get values of this type. -type Key []string - -func (k Key) String() string { - return strings.Join(k, ".") -} - -func (k Key) maybeQuotedAll() string { - var ss []string - for i := range k { - ss = append(ss, k.maybeQuoted(i)) - } - return strings.Join(ss, ".") -} - -func (k Key) maybeQuoted(i int) string { - quote := false - for _, c := range k[i] { - if !isBareKeyChar(c) { - quote = true - break - } - } - if quote { - return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" - } else { - return k[i] - } -} - -func (k Key) add(piece string) Key { - newKey := make(Key, len(k)+1) - copy(newKey, k) - newKey[len(k)] = piece - return newKey -} - -// Keys returns a slice of every key in the TOML data, including key groups. -// Each key is itself a slice, where the first element is the top of the -// hierarchy and the last is the most specific. -// -// The list will have the same order as the keys appeared in the TOML data. -// -// All keys returned are non-empty. -func (md *MetaData) Keys() []Key { - return md.keys -} - -// Undecoded returns all keys that have not been decoded in the order in which -// they appear in the original TOML document. -// -// This includes keys that haven't been decoded because of a Primitive value. -// Once the Primitive value is decoded, the keys will be considered decoded. -// -// Also note that decoding into an empty interface will result in no decoding, -// and so no keys will be considered decoded. -// -// In this sense, the Undecoded keys correspond to keys in the TOML document -// that do not have a concrete type in your representation. -func (md *MetaData) Undecoded() []Key { - undecoded := make([]Key, 0, len(md.keys)) - for _, key := range md.keys { - if !md.decoded[key.String()] { - undecoded = append(undecoded, key) - } - } - return undecoded -} diff --git a/vendor/github.com/BurntSushi/toml/decode_test.go b/vendor/github.com/BurntSushi/toml/decode_test.go deleted file mode 100644 index d337cc43..00000000 --- a/vendor/github.com/BurntSushi/toml/decode_test.go +++ /dev/null @@ -1,1018 +0,0 @@ -package toml - -import ( - "fmt" - "log" - "reflect" - "testing" - "time" -) - -func init() { - log.SetFlags(0) -} - -func TestDecodeSimple(t *testing.T) { - var testSimple = ` -age = 250 -andrew = "gallant" -kait = "brady" -now = 1987-07-05T05:45:00Z -yesOrNo = true -pi = 3.14 -colors = [ - ["red", "green", "blue"], - ["cyan", "magenta", "yellow", "black"], -] - -[My.Cats] -plato = "cat 1" -cauchy = "cat 2" -` - - type cats struct { - Plato string - Cauchy string - } - type simple struct { - Age int - Colors [][]string - Pi float64 - YesOrNo bool - Now time.Time - Andrew string - Kait string - My map[string]cats - } - - var val simple - _, err := Decode(testSimple, &val) - if err != nil { - t.Fatal(err) - } - - now, err := time.Parse("2006-01-02T15:04:05", "1987-07-05T05:45:00") - if err != nil { - panic(err) - } - var answer = simple{ - Age: 250, - Andrew: "gallant", - Kait: "brady", - Now: now, - YesOrNo: true, - Pi: 3.14, - Colors: [][]string{ - {"red", "green", "blue"}, - {"cyan", "magenta", "yellow", "black"}, - }, - My: map[string]cats{ - "Cats": {Plato: "cat 1", Cauchy: "cat 2"}, - }, - } - if !reflect.DeepEqual(val, answer) { - t.Fatalf("Expected\n-----\n%#v\n-----\nbut got\n-----\n%#v\n", - answer, val) - } -} - -func TestDecodeEmbedded(t *testing.T) { - type Dog struct{ Name string } - type Age int - - tests := map[string]struct { - input string - decodeInto interface{} - wantDecoded interface{} - }{ - "embedded struct": { - input: `Name = "milton"`, - decodeInto: &struct{ Dog }{}, - wantDecoded: &struct{ Dog }{Dog{"milton"}}, - }, - "embedded non-nil pointer to struct": { - input: `Name = "milton"`, - decodeInto: &struct{ *Dog }{}, - wantDecoded: &struct{ *Dog }{&Dog{"milton"}}, - }, - "embedded nil pointer to struct": { - input: ``, - decodeInto: &struct{ *Dog }{}, - wantDecoded: &struct{ *Dog }{nil}, - }, - "embedded int": { - input: `Age = -5`, - decodeInto: &struct{ Age }{}, - wantDecoded: &struct{ Age }{-5}, - }, - } - - for label, test := range tests { - _, err := Decode(test.input, test.decodeInto) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(test.wantDecoded, test.decodeInto) { - t.Errorf("%s: want decoded == %+v, got %+v", - label, test.wantDecoded, test.decodeInto) - } - } -} - -func TestDecodeIgnoredFields(t *testing.T) { - type simple struct { - Number int `toml:"-"` - } - const input = ` -Number = 123 -- = 234 -` - var s simple - if _, err := Decode(input, &s); err != nil { - t.Fatal(err) - } - if s.Number != 0 { - t.Errorf("got: %d; want 0", s.Number) - } -} - -func TestTableArrays(t *testing.T) { - var tomlTableArrays = ` -[[albums]] -name = "Born to Run" - - [[albums.songs]] - name = "Jungleland" - - [[albums.songs]] - name = "Meeting Across the River" - -[[albums]] -name = "Born in the USA" - - [[albums.songs]] - name = "Glory Days" - - [[albums.songs]] - name = "Dancing in the Dark" -` - - type Song struct { - Name string - } - - type Album struct { - Name string - Songs []Song - } - - type Music struct { - Albums []Album - } - - expected := Music{[]Album{ - {"Born to Run", []Song{{"Jungleland"}, {"Meeting Across the River"}}}, - {"Born in the USA", []Song{{"Glory Days"}, {"Dancing in the Dark"}}}, - }} - var got Music - if _, err := Decode(tomlTableArrays, &got); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(expected, got) { - t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) - } -} - -// Case insensitive matching tests. -// A bit more comprehensive than needed given the current implementation, -// but implementations change. -// Probably still missing demonstrations of some ugly corner cases regarding -// case insensitive matching and multiple fields. -func TestCase(t *testing.T) { - var caseToml = ` -tOpString = "string" -tOpInt = 1 -tOpFloat = 1.1 -tOpBool = true -tOpdate = 2006-01-02T15:04:05Z -tOparray = [ "array" ] -Match = "i should be in Match only" -MatcH = "i should be in MatcH only" -once = "just once" -[nEst.eD] -nEstedString = "another string" -` - - type InsensitiveEd struct { - NestedString string - } - - type InsensitiveNest struct { - Ed InsensitiveEd - } - - type Insensitive struct { - TopString string - TopInt int - TopFloat float64 - TopBool bool - TopDate time.Time - TopArray []string - Match string - MatcH string - Once string - OncE string - Nest InsensitiveNest - } - - tme, err := time.Parse(time.RFC3339, time.RFC3339[:len(time.RFC3339)-5]) - if err != nil { - panic(err) - } - expected := Insensitive{ - TopString: "string", - TopInt: 1, - TopFloat: 1.1, - TopBool: true, - TopDate: tme, - TopArray: []string{"array"}, - MatcH: "i should be in MatcH only", - Match: "i should be in Match only", - Once: "just once", - OncE: "", - Nest: InsensitiveNest{ - Ed: InsensitiveEd{NestedString: "another string"}, - }, - } - var got Insensitive - if _, err := Decode(caseToml, &got); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(expected, got) { - t.Fatalf("\n%#v\n!=\n%#v\n", expected, got) - } -} - -func TestPointers(t *testing.T) { - type Object struct { - Type string - Description string - } - - type Dict struct { - NamedObject map[string]*Object - BaseObject *Object - Strptr *string - Strptrs []*string - } - s1, s2, s3 := "blah", "abc", "def" - expected := &Dict{ - Strptr: &s1, - Strptrs: []*string{&s2, &s3}, - NamedObject: map[string]*Object{ - "foo": {"FOO", "fooooo!!!"}, - "bar": {"BAR", "ba-ba-ba-ba-barrrr!!!"}, - }, - BaseObject: &Object{"BASE", "da base"}, - } - - ex1 := ` -Strptr = "blah" -Strptrs = ["abc", "def"] - -[NamedObject.foo] -Type = "FOO" -Description = "fooooo!!!" - -[NamedObject.bar] -Type = "BAR" -Description = "ba-ba-ba-ba-barrrr!!!" - -[BaseObject] -Type = "BASE" -Description = "da base" -` - dict := new(Dict) - _, err := Decode(ex1, dict) - if err != nil { - t.Errorf("Decode error: %v", err) - } - if !reflect.DeepEqual(expected, dict) { - t.Fatalf("\n%#v\n!=\n%#v\n", expected, dict) - } -} - -func TestDecodeBadTimestamp(t *testing.T) { - var x struct { - T time.Time - } - for _, s := range []string{ - "T = 123", "T = 2006-01-50T00:00:00Z", "T = 2006-01-30T00:00:00", - } { - if _, err := Decode(s, &x); err == nil { - t.Errorf("Expected invalid DateTime error for %q", s) - } - } -} - -func TestDecodeMultilineStrings(t *testing.T) { - var x struct { - S string - } - const s0 = `s = """ -a b \n c -d e f -"""` - if _, err := Decode(s0, &x); err != nil { - t.Fatal(err) - } - if want := "a b \n c\nd e f\n"; x.S != want { - t.Errorf("got: %q; want: %q", x.S, want) - } - const s1 = `s = """a b c\ -"""` - if _, err := Decode(s1, &x); err != nil { - t.Fatal(err) - } - if want := "a b c"; x.S != want { - t.Errorf("got: %q; want: %q", x.S, want) - } -} - -type sphere struct { - Center [3]float64 - Radius float64 -} - -func TestDecodeSimpleArray(t *testing.T) { - var s1 sphere - if _, err := Decode(`center = [0.0, 1.5, 0.0]`, &s1); err != nil { - t.Fatal(err) - } -} - -func TestDecodeArrayWrongSize(t *testing.T) { - var s1 sphere - if _, err := Decode(`center = [0.1, 2.3]`, &s1); err == nil { - t.Fatal("Expected array type mismatch error") - } -} - -func TestDecodeLargeIntoSmallInt(t *testing.T) { - type table struct { - Value int8 - } - var tab table - if _, err := Decode(`value = 500`, &tab); err == nil { - t.Fatal("Expected integer out-of-bounds error.") - } -} - -func TestDecodeSizedInts(t *testing.T) { - type table struct { - U8 uint8 - U16 uint16 - U32 uint32 - U64 uint64 - U uint - I8 int8 - I16 int16 - I32 int32 - I64 int64 - I int - } - answer := table{1, 1, 1, 1, 1, -1, -1, -1, -1, -1} - toml := ` - u8 = 1 - u16 = 1 - u32 = 1 - u64 = 1 - u = 1 - i8 = -1 - i16 = -1 - i32 = -1 - i64 = -1 - i = -1 - ` - var tab table - if _, err := Decode(toml, &tab); err != nil { - t.Fatal(err.Error()) - } - if answer != tab { - t.Fatalf("Expected %#v but got %#v", answer, tab) - } -} - -func TestUnmarshaler(t *testing.T) { - - var tomlBlob = ` -[dishes.hamboogie] -name = "Hamboogie with fries" -price = 10.99 - -[[dishes.hamboogie.ingredients]] -name = "Bread Bun" - -[[dishes.hamboogie.ingredients]] -name = "Lettuce" - -[[dishes.hamboogie.ingredients]] -name = "Real Beef Patty" - -[[dishes.hamboogie.ingredients]] -name = "Tomato" - -[dishes.eggsalad] -name = "Egg Salad with rice" -price = 3.99 - -[[dishes.eggsalad.ingredients]] -name = "Egg" - -[[dishes.eggsalad.ingredients]] -name = "Mayo" - -[[dishes.eggsalad.ingredients]] -name = "Rice" -` - m := &menu{} - if _, err := Decode(tomlBlob, m); err != nil { - log.Fatal(err) - } - - if len(m.Dishes) != 2 { - t.Log("two dishes should be loaded with UnmarshalTOML()") - t.Errorf("expected %d but got %d", 2, len(m.Dishes)) - } - - eggSalad := m.Dishes["eggsalad"] - if _, ok := interface{}(eggSalad).(dish); !ok { - t.Errorf("expected a dish") - } - - if eggSalad.Name != "Egg Salad with rice" { - t.Errorf("expected the dish to be named 'Egg Salad with rice'") - } - - if len(eggSalad.Ingredients) != 3 { - t.Log("dish should be loaded with UnmarshalTOML()") - t.Errorf("expected %d but got %d", 3, len(eggSalad.Ingredients)) - } - - found := false - for _, i := range eggSalad.Ingredients { - if i.Name == "Rice" { - found = true - break - } - } - if !found { - t.Error("Rice was not loaded in UnmarshalTOML()") - } - - // test on a value - must be passed as * - o := menu{} - if _, err := Decode(tomlBlob, &o); err != nil { - log.Fatal(err) - } - -} - -type menu struct { - Dishes map[string]dish -} - -func (m *menu) UnmarshalTOML(p interface{}) error { - m.Dishes = make(map[string]dish) - data, _ := p.(map[string]interface{}) - dishes := data["dishes"].(map[string]interface{}) - for n, v := range dishes { - if d, ok := v.(map[string]interface{}); ok { - nd := dish{} - nd.UnmarshalTOML(d) - m.Dishes[n] = nd - } else { - return fmt.Errorf("not a dish") - } - } - return nil -} - -type dish struct { - Name string - Price float32 - Ingredients []ingredient -} - -func (d *dish) UnmarshalTOML(p interface{}) error { - data, _ := p.(map[string]interface{}) - d.Name, _ = data["name"].(string) - d.Price, _ = data["price"].(float32) - ingredients, _ := data["ingredients"].([]map[string]interface{}) - for _, e := range ingredients { - n, _ := interface{}(e).(map[string]interface{}) - name, _ := n["name"].(string) - i := ingredient{name} - d.Ingredients = append(d.Ingredients, i) - } - return nil -} - -type ingredient struct { - Name string -} - -func TestDecodeSlices(t *testing.T) { - s := struct{ Test []string }{Test: []string{}} - if _, err := Decode(`Test = ["test"]`, &s); err != nil { - t.Errorf("Error decoding into empty slice: %s", err) - } - s.Test = []string{"a", "b", "c"} - if _, err := Decode(`Test = ["test"]`, &s); err != nil { - t.Errorf("Error decoding into oversized slice: %s", err) - } - if want := []string{"test"}; !reflect.DeepEqual(s.Test, want) { - t.Errorf("Got %v; want %v", s.Test, want) - } -} - -func ExampleMetaData_PrimitiveDecode() { - var md MetaData - var err error - - var tomlBlob = ` -ranking = ["Springsteen", "J Geils"] - -[bands.Springsteen] -started = 1973 -albums = ["Greetings", "WIESS", "Born to Run", "Darkness"] - -[bands."J Geils"] -started = 1970 -albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"] -` - - type band struct { - Started int - Albums []string - } - type classics struct { - Ranking []string - Bands map[string]Primitive - } - - // Do the initial decode. Reflection is delayed on Primitive values. - var music classics - if md, err = Decode(tomlBlob, &music); err != nil { - log.Fatal(err) - } - - // MetaData still includes information on Primitive values. - fmt.Printf("Is `bands.Springsteen` defined? %v\n", - md.IsDefined("bands", "Springsteen")) - - // Decode primitive data into Go values. - for _, artist := range music.Ranking { - // A band is a primitive value, so we need to decode it to get a - // real `band` value. - primValue := music.Bands[artist] - - var aBand band - if err = md.PrimitiveDecode(primValue, &aBand); err != nil { - log.Fatal(err) - } - fmt.Printf("%s started in %d.\n", artist, aBand.Started) - } - // Check to see if there were any fields left undecoded. - // Note that this won't be empty before decoding the Primitive value! - fmt.Printf("Undecoded: %q\n", md.Undecoded()) - - // Output: - // Is `bands.Springsteen` defined? true - // Springsteen started in 1973. - // J Geils started in 1970. - // Undecoded: [] -} - -func ExampleDecode() { - var tomlBlob = ` -# Some comments. -[alpha] -ip = "10.0.0.1" - - [alpha.config] - Ports = [ 8001, 8002 ] - Location = "Toronto" - Created = 1987-07-05T05:45:00Z - -[beta] -ip = "10.0.0.2" - - [beta.config] - Ports = [ 9001, 9002 ] - Location = "New Jersey" - Created = 1887-01-05T05:55:00Z -` - - type serverConfig struct { - Ports []int - Location string - Created time.Time - } - - type server struct { - IP string `toml:"ip,omitempty"` - Config serverConfig `toml:"config"` - } - - type servers map[string]server - - var config servers - if _, err := Decode(tomlBlob, &config); err != nil { - log.Fatal(err) - } - - for _, name := range []string{"alpha", "beta"} { - s := config[name] - fmt.Printf("Server: %s (ip: %s) in %s created on %s\n", - name, s.IP, s.Config.Location, - s.Config.Created.Format("2006-01-02")) - fmt.Printf("Ports: %v\n", s.Config.Ports) - } - - // Output: - // Server: alpha (ip: 10.0.0.1) in Toronto created on 1987-07-05 - // Ports: [8001 8002] - // Server: beta (ip: 10.0.0.2) in New Jersey created on 1887-01-05 - // Ports: [9001 9002] -} - -type duration struct { - time.Duration -} - -func (d *duration) UnmarshalText(text []byte) error { - var err error - d.Duration, err = time.ParseDuration(string(text)) - return err -} - -// Example Unmarshaler shows how to decode TOML strings into your own -// custom data type. -func Example_unmarshaler() { - blob := ` -[[song]] -name = "Thunder Road" -duration = "4m49s" - -[[song]] -name = "Stairway to Heaven" -duration = "8m03s" -` - type song struct { - Name string - Duration duration - } - type songs struct { - Song []song - } - var favorites songs - if _, err := Decode(blob, &favorites); err != nil { - log.Fatal(err) - } - - // Code to implement the TextUnmarshaler interface for `duration`: - // - // type duration struct { - // time.Duration - // } - // - // func (d *duration) UnmarshalText(text []byte) error { - // var err error - // d.Duration, err = time.ParseDuration(string(text)) - // return err - // } - - for _, s := range favorites.Song { - fmt.Printf("%s (%s)\n", s.Name, s.Duration) - } - // Output: - // Thunder Road (4m49s) - // Stairway to Heaven (8m3s) -} - -// Example StrictDecoding shows how to detect whether there are keys in the -// TOML document that weren't decoded into the value given. This is useful -// for returning an error to the user if they've included extraneous fields -// in their configuration. -func Example_strictDecoding() { - var blob = ` -key1 = "value1" -key2 = "value2" -key3 = "value3" -` - type config struct { - Key1 string - Key3 string - } - - var conf config - md, err := Decode(blob, &conf) - if err != nil { - log.Fatal(err) - } - fmt.Printf("Undecoded keys: %q\n", md.Undecoded()) - // Output: - // Undecoded keys: ["key2"] -} - -// Example UnmarshalTOML shows how to implement a struct type that knows how to -// unmarshal itself. The struct must take full responsibility for mapping the -// values passed into the struct. The method may be used with interfaces in a -// struct in cases where the actual type is not known until the data is -// examined. -func Example_unmarshalTOML() { - - var blob = ` -[[parts]] -type = "valve" -id = "valve-1" -size = 1.2 -rating = 4 - -[[parts]] -type = "valve" -id = "valve-2" -size = 2.1 -rating = 5 - -[[parts]] -type = "pipe" -id = "pipe-1" -length = 2.1 -diameter = 12 - -[[parts]] -type = "cable" -id = "cable-1" -length = 12 -rating = 3.1 -` - o := &order{} - err := Unmarshal([]byte(blob), o) - if err != nil { - log.Fatal(err) - } - - fmt.Println(len(o.parts)) - - for _, part := range o.parts { - fmt.Println(part.Name()) - } - - // Code to implement UmarshalJSON. - - // type order struct { - // // NOTE `order.parts` is a private slice of type `part` which is an - // // interface and may only be loaded from toml using the - // // UnmarshalTOML() method of the Umarshaler interface. - // parts parts - // } - - // func (o *order) UnmarshalTOML(data interface{}) error { - - // // NOTE the example below contains detailed type casting to show how - // // the 'data' is retrieved. In operational use, a type cast wrapper - // // may be prefered e.g. - // // - // // func AsMap(v interface{}) (map[string]interface{}, error) { - // // return v.(map[string]interface{}) - // // } - // // - // // resulting in: - // // d, _ := AsMap(data) - // // - - // d, _ := data.(map[string]interface{}) - // parts, _ := d["parts"].([]map[string]interface{}) - - // for _, p := range parts { - - // typ, _ := p["type"].(string) - // id, _ := p["id"].(string) - - // // detect the type of part and handle each case - // switch p["type"] { - // case "valve": - - // size := float32(p["size"].(float64)) - // rating := int(p["rating"].(int64)) - - // valve := &valve{ - // Type: typ, - // ID: id, - // Size: size, - // Rating: rating, - // } - - // o.parts = append(o.parts, valve) - - // case "pipe": - - // length := float32(p["length"].(float64)) - // diameter := int(p["diameter"].(int64)) - - // pipe := &pipe{ - // Type: typ, - // ID: id, - // Length: length, - // Diameter: diameter, - // } - - // o.parts = append(o.parts, pipe) - - // case "cable": - - // length := int(p["length"].(int64)) - // rating := float32(p["rating"].(float64)) - - // cable := &cable{ - // Type: typ, - // ID: id, - // Length: length, - // Rating: rating, - // } - - // o.parts = append(o.parts, cable) - - // } - // } - - // return nil - // } - - // type parts []part - - // type part interface { - // Name() string - // } - - // type valve struct { - // Type string - // ID string - // Size float32 - // Rating int - // } - - // func (v *valve) Name() string { - // return fmt.Sprintf("VALVE: %s", v.ID) - // } - - // type pipe struct { - // Type string - // ID string - // Length float32 - // Diameter int - // } - - // func (p *pipe) Name() string { - // return fmt.Sprintf("PIPE: %s", p.ID) - // } - - // type cable struct { - // Type string - // ID string - // Length int - // Rating float32 - // } - - // func (c *cable) Name() string { - // return fmt.Sprintf("CABLE: %s", c.ID) - // } - - // Output: - // 4 - // VALVE: valve-1 - // VALVE: valve-2 - // PIPE: pipe-1 - // CABLE: cable-1 - -} - -type order struct { - // NOTE `order.parts` is a private slice of type `part` which is an - // interface and may only be loaded from toml using the UnmarshalTOML() - // method of the Umarshaler interface. - parts parts -} - -func (o *order) UnmarshalTOML(data interface{}) error { - - // NOTE the example below contains detailed type casting to show how - // the 'data' is retrieved. In operational use, a type cast wrapper - // may be prefered e.g. - // - // func AsMap(v interface{}) (map[string]interface{}, error) { - // return v.(map[string]interface{}) - // } - // - // resulting in: - // d, _ := AsMap(data) - // - - d, _ := data.(map[string]interface{}) - parts, _ := d["parts"].([]map[string]interface{}) - - for _, p := range parts { - - typ, _ := p["type"].(string) - id, _ := p["id"].(string) - - // detect the type of part and handle each case - switch p["type"] { - case "valve": - - size := float32(p["size"].(float64)) - rating := int(p["rating"].(int64)) - - valve := &valve{ - Type: typ, - ID: id, - Size: size, - Rating: rating, - } - - o.parts = append(o.parts, valve) - - case "pipe": - - length := float32(p["length"].(float64)) - diameter := int(p["diameter"].(int64)) - - pipe := &pipe{ - Type: typ, - ID: id, - Length: length, - Diameter: diameter, - } - - o.parts = append(o.parts, pipe) - - case "cable": - - length := int(p["length"].(int64)) - rating := float32(p["rating"].(float64)) - - cable := &cable{ - Type: typ, - ID: id, - Length: length, - Rating: rating, - } - - o.parts = append(o.parts, cable) - - } - } - - return nil -} - -type parts []part - -type part interface { - Name() string -} - -type valve struct { - Type string - ID string - Size float32 - Rating int -} - -func (v *valve) Name() string { - return fmt.Sprintf("VALVE: %s", v.ID) -} - -type pipe struct { - Type string - ID string - Length float32 - Diameter int -} - -func (p *pipe) Name() string { - return fmt.Sprintf("PIPE: %s", p.ID) -} - -type cable struct { - Type string - ID string - Length int - Rating float32 -} - -func (c *cable) Name() string { - return fmt.Sprintf("CABLE: %s", c.ID) -} diff --git a/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/BurntSushi/toml/doc.go deleted file mode 100644 index fe268000..00000000 --- a/vendor/github.com/BurntSushi/toml/doc.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Package toml provides facilities for decoding and encoding TOML configuration -files via reflection. There is also support for delaying decoding with -the Primitive type, and querying the set of keys in a TOML document with the -MetaData type. - -The specification implemented: https://github.com/mojombo/toml - -The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify -whether a file is a valid TOML document. It can also be used to print the -type of each key in a TOML document. - -Testing - -There are two important types of tests used for this package. The first is -contained inside '*_test.go' files and uses the standard Go unit testing -framework. These tests are primarily devoted to holistically testing the -decoder and encoder. - -The second type of testing is used to verify the implementation's adherence -to the TOML specification. These tests have been factored into their own -project: https://github.com/BurntSushi/toml-test - -The reason the tests are in a separate project is so that they can be used by -any implementation of TOML. Namely, it is language agnostic. -*/ -package toml diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go deleted file mode 100644 index c96216d7..00000000 --- a/vendor/github.com/BurntSushi/toml/encode.go +++ /dev/null @@ -1,562 +0,0 @@ -package toml - -import ( - "bufio" - "errors" - "fmt" - "io" - "reflect" - "sort" - "strconv" - "strings" - "time" -) - -type tomlEncodeError struct{ error } - -var ( - errArrayMixedElementTypes = errors.New( - "can't encode array with mixed element types") - errArrayNilElement = errors.New( - "can't encode array with nil element") - errNonString = errors.New( - "can't encode a map with non-string key type") - errAnonNonStruct = errors.New( - "can't encode an anonymous field that is not a struct") - errArrayNoTable = errors.New( - "TOML array element can't contain a table") - errNoKey = errors.New( - "top-level values must be a Go map or struct") - errAnything = errors.New("") // used in testing -) - -var quotedReplacer = strings.NewReplacer( - "\t", "\\t", - "\n", "\\n", - "\r", "\\r", - "\"", "\\\"", - "\\", "\\\\", -) - -// Encoder controls the encoding of Go values to a TOML document to some -// io.Writer. -// -// The indentation level can be controlled with the Indent field. -type Encoder struct { - // A single indentation level. By default it is two spaces. - Indent string - - // hasWritten is whether we have written any output to w yet. - hasWritten bool - w *bufio.Writer -} - -// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer -// given. By default, a single indentation level is 2 spaces. -func NewEncoder(w io.Writer) *Encoder { - return &Encoder{ - w: bufio.NewWriter(w), - Indent: " ", - } -} - -// Encode writes a TOML representation of the Go value to the underlying -// io.Writer. If the value given cannot be encoded to a valid TOML document, -// then an error is returned. -// -// The mapping between Go values and TOML values should be precisely the same -// as for the Decode* functions. Similarly, the TextMarshaler interface is -// supported by encoding the resulting bytes as strings. (If you want to write -// arbitrary binary data then you will need to use something like base64 since -// TOML does not have any binary types.) -// -// When encoding TOML hashes (i.e., Go maps or structs), keys without any -// sub-hashes are encoded first. -// -// If a Go map is encoded, then its keys are sorted alphabetically for -// deterministic output. More control over this behavior may be provided if -// there is demand for it. -// -// Encoding Go values without a corresponding TOML representation---like map -// types with non-string keys---will cause an error to be returned. Similarly -// for mixed arrays/slices, arrays/slices with nil elements, embedded -// non-struct types and nested slices containing maps or structs. -// (e.g., [][]map[string]string is not allowed but []map[string]string is OK -// and so is []map[string][]string.) -func (enc *Encoder) Encode(v interface{}) error { - rv := eindirect(reflect.ValueOf(v)) - if err := enc.safeEncode(Key([]string{}), rv); err != nil { - return err - } - return enc.w.Flush() -} - -func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) { - defer func() { - if r := recover(); r != nil { - if terr, ok := r.(tomlEncodeError); ok { - err = terr.error - return - } - panic(r) - } - }() - enc.encode(key, rv) - return nil -} - -func (enc *Encoder) encode(key Key, rv reflect.Value) { - // Special case. Time needs to be in ISO8601 format. - // Special case. If we can marshal the type to text, then we used that. - // Basically, this prevents the encoder for handling these types as - // generic structs (or whatever the underlying type of a TextMarshaler is). - switch rv.Interface().(type) { - case time.Time, TextMarshaler: - enc.keyEqElement(key, rv) - return - } - - k := rv.Kind() - switch k { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, - reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, - reflect.Uint64, - reflect.Float32, reflect.Float64, reflect.String, reflect.Bool: - enc.keyEqElement(key, rv) - case reflect.Array, reflect.Slice: - if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) { - enc.eArrayOfTables(key, rv) - } else { - enc.keyEqElement(key, rv) - } - case reflect.Interface: - if rv.IsNil() { - return - } - enc.encode(key, rv.Elem()) - case reflect.Map: - if rv.IsNil() { - return - } - enc.eTable(key, rv) - case reflect.Ptr: - if rv.IsNil() { - return - } - enc.encode(key, rv.Elem()) - case reflect.Struct: - enc.eTable(key, rv) - default: - panic(e("Unsupported type for key '%s': %s", key, k)) - } -} - -// eElement encodes any value that can be an array element (primitives and -// arrays). -func (enc *Encoder) eElement(rv reflect.Value) { - switch v := rv.Interface().(type) { - case time.Time: - // Special case time.Time as a primitive. Has to come before - // TextMarshaler below because time.Time implements - // encoding.TextMarshaler, but we need to always use UTC. - enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z")) - return - case TextMarshaler: - // Special case. Use text marshaler if it's available for this value. - if s, err := v.MarshalText(); err != nil { - encPanic(err) - } else { - enc.writeQuoted(string(s)) - } - return - } - switch rv.Kind() { - case reflect.Bool: - enc.wf(strconv.FormatBool(rv.Bool())) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, - reflect.Int64: - enc.wf(strconv.FormatInt(rv.Int(), 10)) - case reflect.Uint, reflect.Uint8, reflect.Uint16, - reflect.Uint32, reflect.Uint64: - enc.wf(strconv.FormatUint(rv.Uint(), 10)) - case reflect.Float32: - enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32))) - case reflect.Float64: - enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64))) - case reflect.Array, reflect.Slice: - enc.eArrayOrSliceElement(rv) - case reflect.Interface: - enc.eElement(rv.Elem()) - case reflect.String: - enc.writeQuoted(rv.String()) - default: - panic(e("Unexpected primitive type: %s", rv.Kind())) - } -} - -// By the TOML spec, all floats must have a decimal with at least one -// number on either side. -func floatAddDecimal(fstr string) string { - if !strings.Contains(fstr, ".") { - return fstr + ".0" - } - return fstr -} - -func (enc *Encoder) writeQuoted(s string) { - enc.wf("\"%s\"", quotedReplacer.Replace(s)) -} - -func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) { - length := rv.Len() - enc.wf("[") - for i := 0; i < length; i++ { - elem := rv.Index(i) - enc.eElement(elem) - if i != length-1 { - enc.wf(", ") - } - } - enc.wf("]") -} - -func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { - if len(key) == 0 { - encPanic(errNoKey) - } - for i := 0; i < rv.Len(); i++ { - trv := rv.Index(i) - if isNil(trv) { - continue - } - panicIfInvalidKey(key) - enc.newline() - enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll()) - enc.newline() - enc.eMapOrStruct(key, trv) - } -} - -func (enc *Encoder) eTable(key Key, rv reflect.Value) { - panicIfInvalidKey(key) - if len(key) == 1 { - // Output an extra new line between top-level tables. - // (The newline isn't written if nothing else has been written though.) - enc.newline() - } - if len(key) > 0 { - enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll()) - enc.newline() - } - enc.eMapOrStruct(key, rv) -} - -func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) { - switch rv := eindirect(rv); rv.Kind() { - case reflect.Map: - enc.eMap(key, rv) - case reflect.Struct: - enc.eStruct(key, rv) - default: - panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String()) - } -} - -func (enc *Encoder) eMap(key Key, rv reflect.Value) { - rt := rv.Type() - if rt.Key().Kind() != reflect.String { - encPanic(errNonString) - } - - // Sort keys so that we have deterministic output. And write keys directly - // underneath this key first, before writing sub-structs or sub-maps. - var mapKeysDirect, mapKeysSub []string - for _, mapKey := range rv.MapKeys() { - k := mapKey.String() - if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) { - mapKeysSub = append(mapKeysSub, k) - } else { - mapKeysDirect = append(mapKeysDirect, k) - } - } - - var writeMapKeys = func(mapKeys []string) { - sort.Strings(mapKeys) - for _, mapKey := range mapKeys { - mrv := rv.MapIndex(reflect.ValueOf(mapKey)) - if isNil(mrv) { - // Don't write anything for nil fields. - continue - } - enc.encode(key.add(mapKey), mrv) - } - } - writeMapKeys(mapKeysDirect) - writeMapKeys(mapKeysSub) -} - -func (enc *Encoder) eStruct(key Key, rv reflect.Value) { - // Write keys for fields directly under this key first, because if we write - // a field that creates a new table, then all keys under it will be in that - // table (not the one we're writing here). - rt := rv.Type() - var fieldsDirect, fieldsSub [][]int - var addFields func(rt reflect.Type, rv reflect.Value, start []int) - addFields = func(rt reflect.Type, rv reflect.Value, start []int) { - for i := 0; i < rt.NumField(); i++ { - f := rt.Field(i) - // skip unexported fields - if f.PkgPath != "" && !f.Anonymous { - continue - } - frv := rv.Field(i) - if f.Anonymous { - t := f.Type - switch t.Kind() { - case reflect.Struct: - addFields(t, frv, f.Index) - continue - case reflect.Ptr: - if t.Elem().Kind() == reflect.Struct { - if !frv.IsNil() { - addFields(t.Elem(), frv.Elem(), f.Index) - } - continue - } - // Fall through to the normal field encoding logic below - // for non-struct anonymous fields. - } - } - - if typeIsHash(tomlTypeOfGo(frv)) { - fieldsSub = append(fieldsSub, append(start, f.Index...)) - } else { - fieldsDirect = append(fieldsDirect, append(start, f.Index...)) - } - } - } - addFields(rt, rv, nil) - - var writeFields = func(fields [][]int) { - for _, fieldIndex := range fields { - sft := rt.FieldByIndex(fieldIndex) - sf := rv.FieldByIndex(fieldIndex) - if isNil(sf) { - // Don't write anything for nil fields. - continue - } - - keyName := sft.Tag.Get("toml") - if keyName == "-" { - continue - } - if keyName == "" { - keyName = sft.Name - } - - keyName, opts := getOptions(keyName) - if _, ok := opts["omitempty"]; ok && isEmpty(sf) { - continue - } else if _, ok := opts["omitzero"]; ok && isZero(sf) { - continue - } - - enc.encode(key.add(keyName), sf) - } - } - writeFields(fieldsDirect) - writeFields(fieldsSub) -} - -// tomlTypeName returns the TOML type name of the Go value's type. It is -// used to determine whether the types of array elements are mixed (which is -// forbidden). If the Go value is nil, then it is illegal for it to be an array -// element, and valueIsNil is returned as true. - -// Returns the TOML type of a Go value. The type may be `nil`, which means -// no concrete TOML type could be found. -func tomlTypeOfGo(rv reflect.Value) tomlType { - if isNil(rv) || !rv.IsValid() { - return nil - } - switch rv.Kind() { - case reflect.Bool: - return tomlBool - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, - reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, - reflect.Uint64: - return tomlInteger - case reflect.Float32, reflect.Float64: - return tomlFloat - case reflect.Array, reflect.Slice: - if typeEqual(tomlHash, tomlArrayType(rv)) { - return tomlArrayHash - } else { - return tomlArray - } - case reflect.Ptr, reflect.Interface: - return tomlTypeOfGo(rv.Elem()) - case reflect.String: - return tomlString - case reflect.Map: - return tomlHash - case reflect.Struct: - switch rv.Interface().(type) { - case time.Time: - return tomlDatetime - case TextMarshaler: - return tomlString - default: - return tomlHash - } - default: - panic("unexpected reflect.Kind: " + rv.Kind().String()) - } -} - -// tomlArrayType returns the element type of a TOML array. The type returned -// may be nil if it cannot be determined (e.g., a nil slice or a zero length -// slize). This function may also panic if it finds a type that cannot be -// expressed in TOML (such as nil elements, heterogeneous arrays or directly -// nested arrays of tables). -func tomlArrayType(rv reflect.Value) tomlType { - if isNil(rv) || !rv.IsValid() || rv.Len() == 0 { - return nil - } - firstType := tomlTypeOfGo(rv.Index(0)) - if firstType == nil { - encPanic(errArrayNilElement) - } - - rvlen := rv.Len() - for i := 1; i < rvlen; i++ { - elem := rv.Index(i) - switch elemType := tomlTypeOfGo(elem); { - case elemType == nil: - encPanic(errArrayNilElement) - case !typeEqual(firstType, elemType): - encPanic(errArrayMixedElementTypes) - } - } - // If we have a nested array, then we must make sure that the nested - // array contains ONLY primitives. - // This checks arbitrarily nested arrays. - if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) { - nest := tomlArrayType(eindirect(rv.Index(0))) - if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) { - encPanic(errArrayNoTable) - } - } - return firstType -} - -func getOptions(keyName string) (string, map[string]struct{}) { - opts := make(map[string]struct{}) - ss := strings.Split(keyName, ",") - name := ss[0] - if len(ss) > 1 { - for _, opt := range ss { - opts[opt] = struct{}{} - } - } - - return name, opts -} - -func isZero(rv reflect.Value) bool { - switch rv.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if rv.Int() == 0 { - return true - } - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - if rv.Uint() == 0 { - return true - } - case reflect.Float32, reflect.Float64: - if rv.Float() == 0.0 { - return true - } - } - - return false -} - -func isEmpty(rv reflect.Value) bool { - switch rv.Kind() { - case reflect.String: - if len(strings.TrimSpace(rv.String())) == 0 { - return true - } - case reflect.Array, reflect.Slice, reflect.Map: - if rv.Len() == 0 { - return true - } - } - - return false -} - -func (enc *Encoder) newline() { - if enc.hasWritten { - enc.wf("\n") - } -} - -func (enc *Encoder) keyEqElement(key Key, val reflect.Value) { - if len(key) == 0 { - encPanic(errNoKey) - } - panicIfInvalidKey(key) - enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1)) - enc.eElement(val) - enc.newline() -} - -func (enc *Encoder) wf(format string, v ...interface{}) { - if _, err := fmt.Fprintf(enc.w, format, v...); err != nil { - encPanic(err) - } - enc.hasWritten = true -} - -func (enc *Encoder) indentStr(key Key) string { - return strings.Repeat(enc.Indent, len(key)-1) -} - -func encPanic(err error) { - panic(tomlEncodeError{err}) -} - -func eindirect(v reflect.Value) reflect.Value { - switch v.Kind() { - case reflect.Ptr, reflect.Interface: - return eindirect(v.Elem()) - default: - return v - } -} - -func isNil(rv reflect.Value) bool { - switch rv.Kind() { - case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: - return rv.IsNil() - default: - return false - } -} - -func panicIfInvalidKey(key Key) { - for _, k := range key { - if len(k) == 0 { - encPanic(e("Key '%s' is not a valid table name. Key names "+ - "cannot be empty.", key.maybeQuotedAll())) - } - } -} - -func isValidKeyName(s string) bool { - return len(s) != 0 -} diff --git a/vendor/github.com/BurntSushi/toml/encode_test.go b/vendor/github.com/BurntSushi/toml/encode_test.go deleted file mode 100644 index 30f94b1c..00000000 --- a/vendor/github.com/BurntSushi/toml/encode_test.go +++ /dev/null @@ -1,566 +0,0 @@ -package toml - -import ( - "bytes" - "fmt" - "log" - "net" - "testing" - "time" -) - -func TestEncodeRoundTrip(t *testing.T) { - type Config struct { - Age int - Cats []string - Pi float64 - Perfection []int - DOB time.Time - Ipaddress net.IP - } - - var inputs = Config{ - 13, - []string{"one", "two", "three"}, - 3.145, - []int{11, 2, 3, 4}, - time.Now(), - net.ParseIP("192.168.59.254"), - } - - var firstBuffer bytes.Buffer - e := NewEncoder(&firstBuffer) - err := e.Encode(inputs) - if err != nil { - t.Fatal(err) - } - var outputs Config - if _, err := Decode(firstBuffer.String(), &outputs); err != nil { - log.Printf("Could not decode:\n-----\n%s\n-----\n", - firstBuffer.String()) - t.Fatal(err) - } - - // could test each value individually, but I'm lazy - var secondBuffer bytes.Buffer - e2 := NewEncoder(&secondBuffer) - err = e2.Encode(outputs) - if err != nil { - t.Fatal(err) - } - if firstBuffer.String() != secondBuffer.String() { - t.Error( - firstBuffer.String(), - "\n\n is not identical to\n\n", - secondBuffer.String()) - } -} - -// XXX(burntsushi) -// I think these tests probably should be removed. They are good, but they -// ought to be obsolete by toml-test. -func TestEncode(t *testing.T) { - type Embedded struct { - Int int `toml:"_int"` - } - type NonStruct int - - date := time.Date(2014, 5, 11, 20, 30, 40, 0, time.FixedZone("IST", 3600)) - dateStr := "2014-05-11T19:30:40Z" - - tests := map[string]struct { - input interface{} - wantOutput string - wantError error - }{ - "bool field": { - input: struct { - BoolTrue bool - BoolFalse bool - }{true, false}, - wantOutput: "BoolTrue = true\nBoolFalse = false\n", - }, - "int fields": { - input: struct { - Int int - Int8 int8 - Int16 int16 - Int32 int32 - Int64 int64 - }{1, 2, 3, 4, 5}, - wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5\n", - }, - "uint fields": { - input: struct { - Uint uint - Uint8 uint8 - Uint16 uint16 - Uint32 uint32 - Uint64 uint64 - }{1, 2, 3, 4, 5}, - wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" + - "\nUint64 = 5\n", - }, - "float fields": { - input: struct { - Float32 float32 - Float64 float64 - }{1.5, 2.5}, - wantOutput: "Float32 = 1.5\nFloat64 = 2.5\n", - }, - "string field": { - input: struct{ String string }{"foo"}, - wantOutput: "String = \"foo\"\n", - }, - "string field and unexported field": { - input: struct { - String string - unexported int - }{"foo", 0}, - wantOutput: "String = \"foo\"\n", - }, - "datetime field in UTC": { - input: struct{ Date time.Time }{date}, - wantOutput: fmt.Sprintf("Date = %s\n", dateStr), - }, - "datetime field as primitive": { - // Using a map here to fail if isStructOrMap() returns true for - // time.Time. - input: map[string]interface{}{ - "Date": date, - "Int": 1, - }, - wantOutput: fmt.Sprintf("Date = %s\nInt = 1\n", dateStr), - }, - "array fields": { - input: struct { - IntArray0 [0]int - IntArray3 [3]int - }{[0]int{}, [3]int{1, 2, 3}}, - wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]\n", - }, - "slice fields": { - input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{ - nil, []int{}, []int{1, 2, 3}, - }, - wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]\n", - }, - "datetime slices": { - input: struct{ DatetimeSlice []time.Time }{ - []time.Time{date, date}, - }, - wantOutput: fmt.Sprintf("DatetimeSlice = [%s, %s]\n", - dateStr, dateStr), - }, - "nested arrays and slices": { - input: struct { - SliceOfArrays [][2]int - ArrayOfSlices [2][]int - SliceOfArraysOfSlices [][2][]int - ArrayOfSlicesOfArrays [2][][2]int - SliceOfMixedArrays [][2]interface{} - ArrayOfMixedSlices [2][]interface{} - }{ - [][2]int{{1, 2}, {3, 4}}, - [2][]int{{1, 2}, {3, 4}}, - [][2][]int{ - { - {1, 2}, {3, 4}, - }, - { - {5, 6}, {7, 8}, - }, - }, - [2][][2]int{ - { - {1, 2}, {3, 4}, - }, - { - {5, 6}, {7, 8}, - }, - }, - [][2]interface{}{ - {1, 2}, {"a", "b"}, - }, - [2][]interface{}{ - {1, 2}, {"a", "b"}, - }, - }, - wantOutput: `SliceOfArrays = [[1, 2], [3, 4]] -ArrayOfSlices = [[1, 2], [3, 4]] -SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] -ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] -SliceOfMixedArrays = [[1, 2], ["a", "b"]] -ArrayOfMixedSlices = [[1, 2], ["a", "b"]] -`, - }, - "empty slice": { - input: struct{ Empty []interface{} }{[]interface{}{}}, - wantOutput: "Empty = []\n", - }, - "(error) slice with element type mismatch (string and integer)": { - input: struct{ Mixed []interface{} }{[]interface{}{1, "a"}}, - wantError: errArrayMixedElementTypes, - }, - "(error) slice with element type mismatch (integer and float)": { - input: struct{ Mixed []interface{} }{[]interface{}{1, 2.5}}, - wantError: errArrayMixedElementTypes, - }, - "slice with elems of differing Go types, same TOML types": { - input: struct { - MixedInts []interface{} - MixedFloats []interface{} - }{ - []interface{}{ - int(1), int8(2), int16(3), int32(4), int64(5), - uint(1), uint8(2), uint16(3), uint32(4), uint64(5), - }, - []interface{}{float32(1.5), float64(2.5)}, - }, - wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" + - "MixedFloats = [1.5, 2.5]\n", - }, - "(error) slice w/ element type mismatch (one is nested array)": { - input: struct{ Mixed []interface{} }{ - []interface{}{1, []interface{}{2}}, - }, - wantError: errArrayMixedElementTypes, - }, - "(error) slice with 1 nil element": { - input: struct{ NilElement1 []interface{} }{[]interface{}{nil}}, - wantError: errArrayNilElement, - }, - "(error) slice with 1 nil element (and other non-nil elements)": { - input: struct{ NilElement []interface{} }{ - []interface{}{1, nil}, - }, - wantError: errArrayNilElement, - }, - "simple map": { - input: map[string]int{"a": 1, "b": 2}, - wantOutput: "a = 1\nb = 2\n", - }, - "map with interface{} value type": { - input: map[string]interface{}{"a": 1, "b": "c"}, - wantOutput: "a = 1\nb = \"c\"\n", - }, - "map with interface{} value type, some of which are structs": { - input: map[string]interface{}{ - "a": struct{ Int int }{2}, - "b": 1, - }, - wantOutput: "b = 1\n\n[a]\n Int = 2\n", - }, - "nested map": { - input: map[string]map[string]int{ - "a": {"b": 1}, - "c": {"d": 2}, - }, - wantOutput: "[a]\n b = 1\n\n[c]\n d = 2\n", - }, - "nested struct": { - input: struct{ Struct struct{ Int int } }{ - struct{ Int int }{1}, - }, - wantOutput: "[Struct]\n Int = 1\n", - }, - "nested struct and non-struct field": { - input: struct { - Struct struct{ Int int } - Bool bool - }{struct{ Int int }{1}, true}, - wantOutput: "Bool = true\n\n[Struct]\n Int = 1\n", - }, - "2 nested structs": { - input: struct{ Struct1, Struct2 struct{ Int int } }{ - struct{ Int int }{1}, struct{ Int int }{2}, - }, - wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2\n", - }, - "deeply nested structs": { - input: struct { - Struct1, Struct2 struct{ Struct3 *struct{ Int int } } - }{ - struct{ Struct3 *struct{ Int int } }{&struct{ Int int }{1}}, - struct{ Struct3 *struct{ Int int } }{nil}, - }, - wantOutput: "[Struct1]\n [Struct1.Struct3]\n Int = 1" + - "\n\n[Struct2]\n", - }, - "nested struct with nil struct elem": { - input: struct { - Struct struct{ Inner *struct{ Int int } } - }{ - struct{ Inner *struct{ Int int } }{nil}, - }, - wantOutput: "[Struct]\n", - }, - "nested struct with no fields": { - input: struct { - Struct struct{ Inner struct{} } - }{ - struct{ Inner struct{} }{struct{}{}}, - }, - wantOutput: "[Struct]\n [Struct.Inner]\n", - }, - "struct with tags": { - input: struct { - Struct struct { - Int int `toml:"_int"` - } `toml:"_struct"` - Bool bool `toml:"_bool"` - }{ - struct { - Int int `toml:"_int"` - }{1}, true, - }, - wantOutput: "_bool = true\n\n[_struct]\n _int = 1\n", - }, - "embedded struct": { - input: struct{ Embedded }{Embedded{1}}, - wantOutput: "_int = 1\n", - }, - "embedded *struct": { - input: struct{ *Embedded }{&Embedded{1}}, - wantOutput: "_int = 1\n", - }, - "nested embedded struct": { - input: struct { - Struct struct{ Embedded } `toml:"_struct"` - }{struct{ Embedded }{Embedded{1}}}, - wantOutput: "[_struct]\n _int = 1\n", - }, - "nested embedded *struct": { - input: struct { - Struct struct{ *Embedded } `toml:"_struct"` - }{struct{ *Embedded }{&Embedded{1}}}, - wantOutput: "[_struct]\n _int = 1\n", - }, - "embedded non-struct": { - input: struct{ NonStruct }{5}, - wantOutput: "NonStruct = 5\n", - }, - "array of tables": { - input: struct { - Structs []*struct{ Int int } `toml:"struct"` - }{ - []*struct{ Int int }{{1}, {3}}, - }, - wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3\n", - }, - "array of tables order": { - input: map[string]interface{}{ - "map": map[string]interface{}{ - "zero": 5, - "arr": []map[string]int{ - { - "friend": 5, - }, - }, - }, - }, - wantOutput: "[map]\n zero = 5\n\n [[map.arr]]\n friend = 5\n", - }, - "(error) top-level slice": { - input: []struct{ Int int }{{1}, {2}, {3}}, - wantError: errNoKey, - }, - "(error) slice of slice": { - input: struct { - Slices [][]struct{ Int int } - }{ - [][]struct{ Int int }{{{1}}, {{2}}, {{3}}}, - }, - wantError: errArrayNoTable, - }, - "(error) map no string key": { - input: map[int]string{1: ""}, - wantError: errNonString, - }, - "(error) empty key name": { - input: map[string]int{"": 1}, - wantError: errAnything, - }, - "(error) empty map name": { - input: map[string]interface{}{ - "": map[string]int{"v": 1}, - }, - wantError: errAnything, - }, - } - for label, test := range tests { - encodeExpected(t, label, test.input, test.wantOutput, test.wantError) - } -} - -func TestEncodeNestedTableArrays(t *testing.T) { - type song struct { - Name string `toml:"name"` - } - type album struct { - Name string `toml:"name"` - Songs []song `toml:"songs"` - } - type springsteen struct { - Albums []album `toml:"albums"` - } - value := springsteen{ - []album{ - {"Born to Run", - []song{{"Jungleland"}, {"Meeting Across the River"}}}, - {"Born in the USA", - []song{{"Glory Days"}, {"Dancing in the Dark"}}}, - }, - } - expected := `[[albums]] - name = "Born to Run" - - [[albums.songs]] - name = "Jungleland" - - [[albums.songs]] - name = "Meeting Across the River" - -[[albums]] - name = "Born in the USA" - - [[albums.songs]] - name = "Glory Days" - - [[albums.songs]] - name = "Dancing in the Dark" -` - encodeExpected(t, "nested table arrays", value, expected, nil) -} - -func TestEncodeArrayHashWithNormalHashOrder(t *testing.T) { - type Alpha struct { - V int - } - type Beta struct { - V int - } - type Conf struct { - V int - A Alpha - B []Beta - } - - val := Conf{ - V: 1, - A: Alpha{2}, - B: []Beta{{3}}, - } - expected := "V = 1\n\n[A]\n V = 2\n\n[[B]]\n V = 3\n" - encodeExpected(t, "array hash with normal hash order", val, expected, nil) -} - -func TestEncodeWithOmitEmpty(t *testing.T) { - type simple struct { - User string `toml:"user"` - Pass string `toml:"password,omitempty"` - } - - value := simple{"Testing", ""} - expected := fmt.Sprintf("user = %q\n", value.User) - encodeExpected(t, "simple with omitempty, is empty", value, expected, nil) - value.Pass = "some password" - expected = fmt.Sprintf("user = %q\npassword = %q\n", value.User, value.Pass) - encodeExpected(t, "simple with omitempty, not empty", value, expected, nil) -} - -func TestEncodeWithOmitZero(t *testing.T) { - type simple struct { - Number int `toml:"number,omitzero"` - Real float64 `toml:"real,omitzero"` - Unsigned uint `toml:"unsigned,omitzero"` - } - - value := simple{0, 0.0, uint(0)} - expected := "" - - encodeExpected(t, "simple with omitzero, all zero", value, expected, nil) - - value.Number = 10 - value.Real = 20 - value.Unsigned = 5 - expected = `number = 10 -real = 20.0 -unsigned = 5 -` - encodeExpected(t, "simple with omitzero, non-zero", value, expected, nil) -} - -func TestEncodeAnonymousStructPointerField(t *testing.T) { - type Sub struct{} - type simple struct { - *Sub - } - - value := simple{} - expected := "" - encodeExpected(t, "nil anonymous struct pointer field", value, expected, nil) - - value = simple{Sub: &Sub{}} - expected = "" - encodeExpected(t, "non-nil anonymous struct pointer field", value, expected, nil) -} - -func TestEncodeIgnoredFields(t *testing.T) { - type simple struct { - Number int `toml:"-"` - } - value := simple{} - expected := "" - encodeExpected(t, "ignored field", value, expected, nil) -} - -func encodeExpected( - t *testing.T, label string, val interface{}, wantStr string, wantErr error, -) { - var buf bytes.Buffer - enc := NewEncoder(&buf) - err := enc.Encode(val) - if err != wantErr { - if wantErr != nil { - if wantErr == errAnything && err != nil { - return - } - t.Errorf("%s: want Encode error %v, got %v", label, wantErr, err) - } else { - t.Errorf("%s: Encode failed: %s", label, err) - } - } - if err != nil { - return - } - if got := buf.String(); wantStr != got { - t.Errorf("%s: want\n-----\n%q\n-----\nbut got\n-----\n%q\n-----\n", - label, wantStr, got) - } -} - -func ExampleEncoder_Encode() { - date, _ := time.Parse(time.RFC822, "14 Mar 10 18:00 UTC") - var config = map[string]interface{}{ - "date": date, - "counts": []int{1, 1, 2, 3, 5, 8}, - "hash": map[string]string{ - "key1": "val1", - "key2": "val2", - }, - } - buf := new(bytes.Buffer) - if err := NewEncoder(buf).Encode(config); err != nil { - log.Fatal(err) - } - fmt.Println(buf.String()) - - // Output: - // counts = [1, 1, 2, 3, 5, 8] - // date = 2010-03-14T18:00:00Z - // - // [hash] - // key1 = "val1" - // key2 = "val2" -} diff --git a/vendor/github.com/BurntSushi/toml/encoding_types.go b/vendor/github.com/BurntSushi/toml/encoding_types.go deleted file mode 100644 index d36e1dd6..00000000 --- a/vendor/github.com/BurntSushi/toml/encoding_types.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build go1.2 - -package toml - -// In order to support Go 1.1, we define our own TextMarshaler and -// TextUnmarshaler types. For Go 1.2+, we just alias them with the -// standard library interfaces. - -import ( - "encoding" -) - -// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here -// so that Go 1.1 can be supported. -type TextMarshaler encoding.TextMarshaler - -// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined -// here so that Go 1.1 can be supported. -type TextUnmarshaler encoding.TextUnmarshaler diff --git a/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go b/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go deleted file mode 100644 index e8d503d0..00000000 --- a/vendor/github.com/BurntSushi/toml/encoding_types_1.1.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build !go1.2 - -package toml - -// These interfaces were introduced in Go 1.2, so we add them manually when -// compiling for Go 1.1. - -// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here -// so that Go 1.1 can be supported. -type TextMarshaler interface { - MarshalText() (text []byte, err error) -} - -// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined -// here so that Go 1.1 can be supported. -type TextUnmarshaler interface { - UnmarshalText(text []byte) error -} diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go deleted file mode 100644 index 9b20b3a8..00000000 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ /dev/null @@ -1,871 +0,0 @@ -package toml - -import ( - "fmt" - "strings" - "unicode/utf8" -) - -type itemType int - -const ( - itemError itemType = iota - itemNIL // used in the parser to indicate no type - itemEOF - itemText - itemString - itemRawString - itemMultilineString - itemRawMultilineString - itemBool - itemInteger - itemFloat - itemDatetime - itemArray // the start of an array - itemArrayEnd - itemTableStart - itemTableEnd - itemArrayTableStart - itemArrayTableEnd - itemKeyStart - itemCommentStart -) - -const ( - eof = 0 - tableStart = '[' - tableEnd = ']' - arrayTableStart = '[' - arrayTableEnd = ']' - tableSep = '.' - keySep = '=' - arrayStart = '[' - arrayEnd = ']' - arrayValTerm = ',' - commentStart = '#' - stringStart = '"' - stringEnd = '"' - rawStringStart = '\'' - rawStringEnd = '\'' -) - -type stateFn func(lx *lexer) stateFn - -type lexer struct { - input string - start int - pos int - width int - line int - state stateFn - items chan item - - // A stack of state functions used to maintain context. - // The idea is to reuse parts of the state machine in various places. - // For example, values can appear at the top level or within arbitrarily - // nested arrays. The last state on the stack is used after a value has - // been lexed. Similarly for comments. - stack []stateFn -} - -type item struct { - typ itemType - val string - line int -} - -func (lx *lexer) nextItem() item { - for { - select { - case item := <-lx.items: - return item - default: - lx.state = lx.state(lx) - } - } -} - -func lex(input string) *lexer { - lx := &lexer{ - input: input + "\n", - state: lexTop, - line: 1, - items: make(chan item, 10), - stack: make([]stateFn, 0, 10), - } - return lx -} - -func (lx *lexer) push(state stateFn) { - lx.stack = append(lx.stack, state) -} - -func (lx *lexer) pop() stateFn { - if len(lx.stack) == 0 { - return lx.errorf("BUG in lexer: no states to pop.") - } - last := lx.stack[len(lx.stack)-1] - lx.stack = lx.stack[0 : len(lx.stack)-1] - return last -} - -func (lx *lexer) current() string { - return lx.input[lx.start:lx.pos] -} - -func (lx *lexer) emit(typ itemType) { - lx.items <- item{typ, lx.current(), lx.line} - lx.start = lx.pos -} - -func (lx *lexer) emitTrim(typ itemType) { - lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line} - lx.start = lx.pos -} - -func (lx *lexer) next() (r rune) { - if lx.pos >= len(lx.input) { - lx.width = 0 - return eof - } - - if lx.input[lx.pos] == '\n' { - lx.line++ - } - r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) - lx.pos += lx.width - return r -} - -// ignore skips over the pending input before this point. -func (lx *lexer) ignore() { - lx.start = lx.pos -} - -// backup steps back one rune. Can be called only once per call of next. -func (lx *lexer) backup() { - lx.pos -= lx.width - if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { - lx.line-- - } -} - -// accept consumes the next rune if it's equal to `valid`. -func (lx *lexer) accept(valid rune) bool { - if lx.next() == valid { - return true - } - lx.backup() - return false -} - -// peek returns but does not consume the next rune in the input. -func (lx *lexer) peek() rune { - r := lx.next() - lx.backup() - return r -} - -// errorf stops all lexing by emitting an error and returning `nil`. -// Note that any value that is a character is escaped if it's a special -// character (new lines, tabs, etc.). -func (lx *lexer) errorf(format string, values ...interface{}) stateFn { - lx.items <- item{ - itemError, - fmt.Sprintf(format, values...), - lx.line, - } - return nil -} - -// lexTop consumes elements at the top level of TOML data. -func lexTop(lx *lexer) stateFn { - r := lx.next() - if isWhitespace(r) || isNL(r) { - return lexSkip(lx, lexTop) - } - - switch r { - case commentStart: - lx.push(lexTop) - return lexCommentStart - case tableStart: - return lexTableStart - case eof: - if lx.pos > lx.start { - return lx.errorf("Unexpected EOF.") - } - lx.emit(itemEOF) - return nil - } - - // At this point, the only valid item can be a key, so we back up - // and let the key lexer do the rest. - lx.backup() - lx.push(lexTopEnd) - return lexKeyStart -} - -// lexTopEnd is entered whenever a top-level item has been consumed. (A value -// or a table.) It must see only whitespace, and will turn back to lexTop -// upon a new line. If it sees EOF, it will quit the lexer successfully. -func lexTopEnd(lx *lexer) stateFn { - r := lx.next() - switch { - case r == commentStart: - // a comment will read to a new line for us. - lx.push(lexTop) - return lexCommentStart - case isWhitespace(r): - return lexTopEnd - case isNL(r): - lx.ignore() - return lexTop - case r == eof: - lx.ignore() - return lexTop - } - return lx.errorf("Expected a top-level item to end with a new line, "+ - "comment or EOF, but got %q instead.", r) -} - -// lexTable lexes the beginning of a table. Namely, it makes sure that -// it starts with a character other than '.' and ']'. -// It assumes that '[' has already been consumed. -// It also handles the case that this is an item in an array of tables. -// e.g., '[[name]]'. -func lexTableStart(lx *lexer) stateFn { - if lx.peek() == arrayTableStart { - lx.next() - lx.emit(itemArrayTableStart) - lx.push(lexArrayTableEnd) - } else { - lx.emit(itemTableStart) - lx.push(lexTableEnd) - } - return lexTableNameStart -} - -func lexTableEnd(lx *lexer) stateFn { - lx.emit(itemTableEnd) - return lexTopEnd -} - -func lexArrayTableEnd(lx *lexer) stateFn { - if r := lx.next(); r != arrayTableEnd { - return lx.errorf("Expected end of table array name delimiter %q, "+ - "but got %q instead.", arrayTableEnd, r) - } - lx.emit(itemArrayTableEnd) - return lexTopEnd -} - -func lexTableNameStart(lx *lexer) stateFn { - switch r := lx.peek(); { - case r == tableEnd || r == eof: - return lx.errorf("Unexpected end of table name. (Table names cannot " + - "be empty.)") - case r == tableSep: - return lx.errorf("Unexpected table separator. (Table names cannot " + - "be empty.)") - case r == stringStart || r == rawStringStart: - lx.ignore() - lx.push(lexTableNameEnd) - return lexValue // reuse string lexing - default: - return lexBareTableName - } -} - -// lexTableName lexes the name of a table. It assumes that at least one -// valid character for the table has already been read. -func lexBareTableName(lx *lexer) stateFn { - switch r := lx.next(); { - case isBareKeyChar(r): - return lexBareTableName - case r == tableSep || r == tableEnd: - lx.backup() - lx.emitTrim(itemText) - return lexTableNameEnd - default: - return lx.errorf("Bare keys cannot contain %q.", r) - } -} - -// lexTableNameEnd reads the end of a piece of a table name, optionally -// consuming whitespace. -func lexTableNameEnd(lx *lexer) stateFn { - switch r := lx.next(); { - case isWhitespace(r): - return lexTableNameEnd - case r == tableSep: - lx.ignore() - return lexTableNameStart - case r == tableEnd: - return lx.pop() - default: - return lx.errorf("Expected '.' or ']' to end table name, but got %q "+ - "instead.", r) - } -} - -// lexKeyStart consumes a key name up until the first non-whitespace character. -// lexKeyStart will ignore whitespace. -func lexKeyStart(lx *lexer) stateFn { - r := lx.peek() - switch { - case r == keySep: - return lx.errorf("Unexpected key separator %q.", keySep) - case isWhitespace(r) || isNL(r): - lx.next() - return lexSkip(lx, lexKeyStart) - case r == stringStart || r == rawStringStart: - lx.ignore() - lx.emit(itemKeyStart) - lx.push(lexKeyEnd) - return lexValue // reuse string lexing - default: - lx.ignore() - lx.emit(itemKeyStart) - return lexBareKey - } -} - -// lexBareKey consumes the text of a bare key. Assumes that the first character -// (which is not whitespace) has not yet been consumed. -func lexBareKey(lx *lexer) stateFn { - switch r := lx.next(); { - case isBareKeyChar(r): - return lexBareKey - case isWhitespace(r): - lx.emitTrim(itemText) - return lexKeyEnd - case r == keySep: - lx.backup() - lx.emitTrim(itemText) - return lexKeyEnd - default: - return lx.errorf("Bare keys cannot contain %q.", r) - } -} - -// lexKeyEnd consumes the end of a key and trims whitespace (up to the key -// separator). -func lexKeyEnd(lx *lexer) stateFn { - switch r := lx.next(); { - case r == keySep: - return lexSkip(lx, lexValue) - case isWhitespace(r): - return lexSkip(lx, lexKeyEnd) - default: - return lx.errorf("Expected key separator %q, but got %q instead.", - keySep, r) - } -} - -// lexValue starts the consumption of a value anywhere a value is expected. -// lexValue will ignore whitespace. -// After a value is lexed, the last state on the next is popped and returned. -func lexValue(lx *lexer) stateFn { - // We allow whitespace to precede a value, but NOT new lines. - // In array syntax, the array states are responsible for ignoring new - // lines. - r := lx.next() - if isWhitespace(r) { - return lexSkip(lx, lexValue) - } - - switch { - case r == arrayStart: - lx.ignore() - lx.emit(itemArray) - return lexArrayValue - case r == stringStart: - if lx.accept(stringStart) { - if lx.accept(stringStart) { - lx.ignore() // Ignore """ - return lexMultilineString - } - lx.backup() - } - lx.ignore() // ignore the '"' - return lexString - case r == rawStringStart: - if lx.accept(rawStringStart) { - if lx.accept(rawStringStart) { - lx.ignore() // Ignore """ - return lexMultilineRawString - } - lx.backup() - } - lx.ignore() // ignore the "'" - return lexRawString - case r == 't': - return lexTrue - case r == 'f': - return lexFalse - case r == '-': - return lexNumberStart - case isDigit(r): - lx.backup() // avoid an extra state and use the same as above - return lexNumberOrDateStart - case r == '.': // special error case, be kind to users - return lx.errorf("Floats must start with a digit, not '.'.") - } - return lx.errorf("Expected value but found %q instead.", r) -} - -// lexArrayValue consumes one value in an array. It assumes that '[' or ',' -// have already been consumed. All whitespace and new lines are ignored. -func lexArrayValue(lx *lexer) stateFn { - r := lx.next() - switch { - case isWhitespace(r) || isNL(r): - return lexSkip(lx, lexArrayValue) - case r == commentStart: - lx.push(lexArrayValue) - return lexCommentStart - case r == arrayValTerm: - return lx.errorf("Unexpected array value terminator %q.", - arrayValTerm) - case r == arrayEnd: - return lexArrayEnd - } - - lx.backup() - lx.push(lexArrayValueEnd) - return lexValue -} - -// lexArrayValueEnd consumes the cruft between values of an array. Namely, -// it ignores whitespace and expects either a ',' or a ']'. -func lexArrayValueEnd(lx *lexer) stateFn { - r := lx.next() - switch { - case isWhitespace(r) || isNL(r): - return lexSkip(lx, lexArrayValueEnd) - case r == commentStart: - lx.push(lexArrayValueEnd) - return lexCommentStart - case r == arrayValTerm: - lx.ignore() - return lexArrayValue // move on to the next value - case r == arrayEnd: - return lexArrayEnd - } - return lx.errorf("Expected an array value terminator %q or an array "+ - "terminator %q, but got %q instead.", arrayValTerm, arrayEnd, r) -} - -// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has -// just been consumed. -func lexArrayEnd(lx *lexer) stateFn { - lx.ignore() - lx.emit(itemArrayEnd) - return lx.pop() -} - -// lexString consumes the inner contents of a string. It assumes that the -// beginning '"' has already been consumed and ignored. -func lexString(lx *lexer) stateFn { - r := lx.next() - switch { - case isNL(r): - return lx.errorf("Strings cannot contain new lines.") - case r == '\\': - lx.push(lexString) - return lexStringEscape - case r == stringEnd: - lx.backup() - lx.emit(itemString) - lx.next() - lx.ignore() - return lx.pop() - } - return lexString -} - -// lexMultilineString consumes the inner contents of a string. It assumes that -// the beginning '"""' has already been consumed and ignored. -func lexMultilineString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '\\': - return lexMultilineStringEscape - case r == stringEnd: - if lx.accept(stringEnd) { - if lx.accept(stringEnd) { - lx.backup() - lx.backup() - lx.backup() - lx.emit(itemMultilineString) - lx.next() - lx.next() - lx.next() - lx.ignore() - return lx.pop() - } - lx.backup() - } - } - return lexMultilineString -} - -// lexRawString consumes a raw string. Nothing can be escaped in such a string. -// It assumes that the beginning "'" has already been consumed and ignored. -func lexRawString(lx *lexer) stateFn { - r := lx.next() - switch { - case isNL(r): - return lx.errorf("Strings cannot contain new lines.") - case r == rawStringEnd: - lx.backup() - lx.emit(itemRawString) - lx.next() - lx.ignore() - return lx.pop() - } - return lexRawString -} - -// lexMultilineRawString consumes a raw string. Nothing can be escaped in such -// a string. It assumes that the beginning "'" has already been consumed and -// ignored. -func lexMultilineRawString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == rawStringEnd: - if lx.accept(rawStringEnd) { - if lx.accept(rawStringEnd) { - lx.backup() - lx.backup() - lx.backup() - lx.emit(itemRawMultilineString) - lx.next() - lx.next() - lx.next() - lx.ignore() - return lx.pop() - } - lx.backup() - } - } - return lexMultilineRawString -} - -// lexMultilineStringEscape consumes an escaped character. It assumes that the -// preceding '\\' has already been consumed. -func lexMultilineStringEscape(lx *lexer) stateFn { - // Handle the special case first: - if isNL(lx.next()) { - return lexMultilineString - } else { - lx.backup() - lx.push(lexMultilineString) - return lexStringEscape(lx) - } -} - -func lexStringEscape(lx *lexer) stateFn { - r := lx.next() - switch r { - case 'b': - fallthrough - case 't': - fallthrough - case 'n': - fallthrough - case 'f': - fallthrough - case 'r': - fallthrough - case '"': - fallthrough - case '\\': - return lx.pop() - case 'u': - return lexShortUnicodeEscape - case 'U': - return lexLongUnicodeEscape - } - return lx.errorf("Invalid escape character %q. Only the following "+ - "escape characters are allowed: "+ - "\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, "+ - "\\uXXXX and \\UXXXXXXXX.", r) -} - -func lexShortUnicodeEscape(lx *lexer) stateFn { - var r rune - for i := 0; i < 4; i++ { - r = lx.next() - if !isHexadecimal(r) { - return lx.errorf("Expected four hexadecimal digits after '\\u', "+ - "but got '%s' instead.", lx.current()) - } - } - return lx.pop() -} - -func lexLongUnicodeEscape(lx *lexer) stateFn { - var r rune - for i := 0; i < 8; i++ { - r = lx.next() - if !isHexadecimal(r) { - return lx.errorf("Expected eight hexadecimal digits after '\\U', "+ - "but got '%s' instead.", lx.current()) - } - } - return lx.pop() -} - -// lexNumberOrDateStart consumes either a (positive) integer, float or -// datetime. It assumes that NO negative sign has been consumed. -func lexNumberOrDateStart(lx *lexer) stateFn { - r := lx.next() - if !isDigit(r) { - if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) - } - } - return lexNumberOrDate -} - -// lexNumberOrDate consumes either a (positive) integer, float or datetime. -func lexNumberOrDate(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '-': - if lx.pos-lx.start != 5 { - return lx.errorf("All ISO8601 dates must be in full Zulu form.") - } - return lexDateAfterYear - case isDigit(r): - return lexNumberOrDate - case r == '.': - return lexFloatStart - } - - lx.backup() - lx.emit(itemInteger) - return lx.pop() -} - -// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. -// It assumes that "YYYY-" has already been consumed. -func lexDateAfterYear(lx *lexer) stateFn { - formats := []rune{ - // digits are '0'. - // everything else is direct equality. - '0', '0', '-', '0', '0', - 'T', - '0', '0', ':', '0', '0', ':', '0', '0', - 'Z', - } - for _, f := range formats { - r := lx.next() - if f == '0' { - if !isDigit(r) { - return lx.errorf("Expected digit in ISO8601 datetime, "+ - "but found %q instead.", r) - } - } else if f != r { - return lx.errorf("Expected %q in ISO8601 datetime, "+ - "but found %q instead.", f, r) - } - } - lx.emit(itemDatetime) - return lx.pop() -} - -// lexNumberStart consumes either an integer or a float. It assumes that -// a negative sign has already been read, but that *no* digits have been -// consumed. lexNumberStart will move to the appropriate integer or float -// states. -func lexNumberStart(lx *lexer) stateFn { - // we MUST see a digit. Even floats have to start with a digit. - r := lx.next() - if !isDigit(r) { - if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) - } - } - return lexNumber -} - -// lexNumber consumes an integer or a float after seeing the first digit. -func lexNumber(lx *lexer) stateFn { - r := lx.next() - switch { - case isDigit(r): - return lexNumber - case r == '.': - return lexFloatStart - } - - lx.backup() - lx.emit(itemInteger) - return lx.pop() -} - -// lexFloatStart starts the consumption of digits of a float after a '.'. -// Namely, at least one digit is required. -func lexFloatStart(lx *lexer) stateFn { - r := lx.next() - if !isDigit(r) { - return lx.errorf("Floats must have a digit after the '.', but got "+ - "%q instead.", r) - } - return lexFloat -} - -// lexFloat consumes the digits of a float after a '.'. -// Assumes that one digit has been consumed after a '.' already. -func lexFloat(lx *lexer) stateFn { - r := lx.next() - if isDigit(r) { - return lexFloat - } - - lx.backup() - lx.emit(itemFloat) - return lx.pop() -} - -// lexConst consumes the s[1:] in s. It assumes that s[0] has already been -// consumed. -func lexConst(lx *lexer, s string) stateFn { - for i := range s[1:] { - if r := lx.next(); r != rune(s[i+1]) { - return lx.errorf("Expected %q, but found %q instead.", s[:i+1], - s[:i]+string(r)) - } - } - return nil -} - -// lexTrue consumes the "rue" in "true". It assumes that 't' has already -// been consumed. -func lexTrue(lx *lexer) stateFn { - if fn := lexConst(lx, "true"); fn != nil { - return fn - } - lx.emit(itemBool) - return lx.pop() -} - -// lexFalse consumes the "alse" in "false". It assumes that 'f' has already -// been consumed. -func lexFalse(lx *lexer) stateFn { - if fn := lexConst(lx, "false"); fn != nil { - return fn - } - lx.emit(itemBool) - return lx.pop() -} - -// lexCommentStart begins the lexing of a comment. It will emit -// itemCommentStart and consume no characters, passing control to lexComment. -func lexCommentStart(lx *lexer) stateFn { - lx.ignore() - lx.emit(itemCommentStart) - return lexComment -} - -// lexComment lexes an entire comment. It assumes that '#' has been consumed. -// It will consume *up to* the first new line character, and pass control -// back to the last state on the stack. -func lexComment(lx *lexer) stateFn { - r := lx.peek() - if isNL(r) || r == eof { - lx.emit(itemText) - return lx.pop() - } - lx.next() - return lexComment -} - -// lexSkip ignores all slurped input and moves on to the next state. -func lexSkip(lx *lexer, nextState stateFn) stateFn { - return func(lx *lexer) stateFn { - lx.ignore() - return nextState - } -} - -// isWhitespace returns true if `r` is a whitespace character according -// to the spec. -func isWhitespace(r rune) bool { - return r == '\t' || r == ' ' -} - -func isNL(r rune) bool { - return r == '\n' || r == '\r' -} - -func isDigit(r rune) bool { - return r >= '0' && r <= '9' -} - -func isHexadecimal(r rune) bool { - return (r >= '0' && r <= '9') || - (r >= 'a' && r <= 'f') || - (r >= 'A' && r <= 'F') -} - -func isBareKeyChar(r rune) bool { - return (r >= 'A' && r <= 'Z') || - (r >= 'a' && r <= 'z') || - (r >= '0' && r <= '9') || - r == '_' || - r == '-' -} - -func (itype itemType) String() string { - switch itype { - case itemError: - return "Error" - case itemNIL: - return "NIL" - case itemEOF: - return "EOF" - case itemText: - return "Text" - case itemString: - return "String" - case itemRawString: - return "String" - case itemMultilineString: - return "String" - case itemRawMultilineString: - return "String" - case itemBool: - return "Bool" - case itemInteger: - return "Integer" - case itemFloat: - return "Float" - case itemDatetime: - return "DateTime" - case itemTableStart: - return "TableStart" - case itemTableEnd: - return "TableEnd" - case itemKeyStart: - return "KeyStart" - case itemArray: - return "Array" - case itemArrayEnd: - return "ArrayEnd" - case itemCommentStart: - return "CommentStart" - } - panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype))) -} - -func (item item) String() string { - return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val) -} diff --git a/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/BurntSushi/toml/parse.go deleted file mode 100644 index 6a82e84f..00000000 --- a/vendor/github.com/BurntSushi/toml/parse.go +++ /dev/null @@ -1,493 +0,0 @@ -package toml - -import ( - "fmt" - "log" - "strconv" - "strings" - "time" - "unicode" - "unicode/utf8" -) - -type parser struct { - mapping map[string]interface{} - types map[string]tomlType - lx *lexer - - // A list of keys in the order that they appear in the TOML data. - ordered []Key - - // the full key for the current hash in scope - context Key - - // the base key name for everything except hashes - currentKey string - - // rough approximation of line number - approxLine int - - // A map of 'key.group.names' to whether they were created implicitly. - implicits map[string]bool -} - -type parseError string - -func (pe parseError) Error() string { - return string(pe) -} - -func parse(data string) (p *parser, err error) { - defer func() { - if r := recover(); r != nil { - var ok bool - if err, ok = r.(parseError); ok { - return - } - panic(r) - } - }() - - p = &parser{ - mapping: make(map[string]interface{}), - types: make(map[string]tomlType), - lx: lex(data), - ordered: make([]Key, 0), - implicits: make(map[string]bool), - } - for { - item := p.next() - if item.typ == itemEOF { - break - } - p.topLevel(item) - } - - return p, nil -} - -func (p *parser) panicf(format string, v ...interface{}) { - msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s", - p.approxLine, p.current(), fmt.Sprintf(format, v...)) - panic(parseError(msg)) -} - -func (p *parser) next() item { - it := p.lx.nextItem() - if it.typ == itemError { - p.panicf("%s", it.val) - } - return it -} - -func (p *parser) bug(format string, v ...interface{}) { - log.Panicf("BUG: %s\n\n", fmt.Sprintf(format, v...)) -} - -func (p *parser) expect(typ itemType) item { - it := p.next() - p.assertEqual(typ, it.typ) - return it -} - -func (p *parser) assertEqual(expected, got itemType) { - if expected != got { - p.bug("Expected '%s' but got '%s'.", expected, got) - } -} - -func (p *parser) topLevel(item item) { - switch item.typ { - case itemCommentStart: - p.approxLine = item.line - p.expect(itemText) - case itemTableStart: - kg := p.next() - p.approxLine = kg.line - - var key Key - for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() { - key = append(key, p.keyString(kg)) - } - p.assertEqual(itemTableEnd, kg.typ) - - p.establishContext(key, false) - p.setType("", tomlHash) - p.ordered = append(p.ordered, key) - case itemArrayTableStart: - kg := p.next() - p.approxLine = kg.line - - var key Key - for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() { - key = append(key, p.keyString(kg)) - } - p.assertEqual(itemArrayTableEnd, kg.typ) - - p.establishContext(key, true) - p.setType("", tomlArrayHash) - p.ordered = append(p.ordered, key) - case itemKeyStart: - kname := p.next() - p.approxLine = kname.line - p.currentKey = p.keyString(kname) - - val, typ := p.value(p.next()) - p.setValue(p.currentKey, val) - p.setType(p.currentKey, typ) - p.ordered = append(p.ordered, p.context.add(p.currentKey)) - p.currentKey = "" - default: - p.bug("Unexpected type at top level: %s", item.typ) - } -} - -// Gets a string for a key (or part of a key in a table name). -func (p *parser) keyString(it item) string { - switch it.typ { - case itemText: - return it.val - case itemString, itemMultilineString, - itemRawString, itemRawMultilineString: - s, _ := p.value(it) - return s.(string) - default: - p.bug("Unexpected key type: %s", it.typ) - panic("unreachable") - } -} - -// value translates an expected value from the lexer into a Go value wrapped -// as an empty interface. -func (p *parser) value(it item) (interface{}, tomlType) { - switch it.typ { - case itemString: - return p.replaceEscapes(it.val), p.typeOfPrimitive(it) - case itemMultilineString: - trimmed := stripFirstNewline(stripEscapedWhitespace(it.val)) - return p.replaceEscapes(trimmed), p.typeOfPrimitive(it) - case itemRawString: - return it.val, p.typeOfPrimitive(it) - case itemRawMultilineString: - return stripFirstNewline(it.val), p.typeOfPrimitive(it) - case itemBool: - switch it.val { - case "true": - return true, p.typeOfPrimitive(it) - case "false": - return false, p.typeOfPrimitive(it) - } - p.bug("Expected boolean value, but got '%s'.", it.val) - case itemInteger: - num, err := strconv.ParseInt(it.val, 10, 64) - if err != nil { - // See comment below for floats describing why we make a - // distinction between a bug and a user error. - if e, ok := err.(*strconv.NumError); ok && - e.Err == strconv.ErrRange { - - p.panicf("Integer '%s' is out of the range of 64-bit "+ - "signed integers.", it.val) - } else { - p.bug("Expected integer value, but got '%s'.", it.val) - } - } - return num, p.typeOfPrimitive(it) - case itemFloat: - num, err := strconv.ParseFloat(it.val, 64) - if err != nil { - // Distinguish float values. Normally, it'd be a bug if the lexer - // provides an invalid float, but it's possible that the float is - // out of range of valid values (which the lexer cannot determine). - // So mark the former as a bug but the latter as a legitimate user - // error. - // - // This is also true for integers. - if e, ok := err.(*strconv.NumError); ok && - e.Err == strconv.ErrRange { - - p.panicf("Float '%s' is out of the range of 64-bit "+ - "IEEE-754 floating-point numbers.", it.val) - } else { - p.bug("Expected float value, but got '%s'.", it.val) - } - } - return num, p.typeOfPrimitive(it) - case itemDatetime: - t, err := time.Parse("2006-01-02T15:04:05Z", it.val) - if err != nil { - p.panicf("Invalid RFC3339 Zulu DateTime: '%s'.", it.val) - } - return t, p.typeOfPrimitive(it) - case itemArray: - array := make([]interface{}, 0) - types := make([]tomlType, 0) - - for it = p.next(); it.typ != itemArrayEnd; it = p.next() { - if it.typ == itemCommentStart { - p.expect(itemText) - continue - } - - val, typ := p.value(it) - array = append(array, val) - types = append(types, typ) - } - return array, p.typeOfArray(types) - } - p.bug("Unexpected value type: %s", it.typ) - panic("unreachable") -} - -// establishContext sets the current context of the parser, -// where the context is either a hash or an array of hashes. Which one is -// set depends on the value of the `array` parameter. -// -// Establishing the context also makes sure that the key isn't a duplicate, and -// will create implicit hashes automatically. -func (p *parser) establishContext(key Key, array bool) { - var ok bool - - // Always start at the top level and drill down for our context. - hashContext := p.mapping - keyContext := make(Key, 0) - - // We only need implicit hashes for key[0:-1] - for _, k := range key[0 : len(key)-1] { - _, ok = hashContext[k] - keyContext = append(keyContext, k) - - // No key? Make an implicit hash and move on. - if !ok { - p.addImplicit(keyContext) - hashContext[k] = make(map[string]interface{}) - } - - // If the hash context is actually an array of tables, then set - // the hash context to the last element in that array. - // - // Otherwise, it better be a table, since this MUST be a key group (by - // virtue of it not being the last element in a key). - switch t := hashContext[k].(type) { - case []map[string]interface{}: - hashContext = t[len(t)-1] - case map[string]interface{}: - hashContext = t - default: - p.panicf("Key '%s' was already created as a hash.", keyContext) - } - } - - p.context = keyContext - if array { - // If this is the first element for this array, then allocate a new - // list of tables for it. - k := key[len(key)-1] - if _, ok := hashContext[k]; !ok { - hashContext[k] = make([]map[string]interface{}, 0, 5) - } - - // Add a new table. But make sure the key hasn't already been used - // for something else. - if hash, ok := hashContext[k].([]map[string]interface{}); ok { - hashContext[k] = append(hash, make(map[string]interface{})) - } else { - p.panicf("Key '%s' was already created and cannot be used as "+ - "an array.", keyContext) - } - } else { - p.setValue(key[len(key)-1], make(map[string]interface{})) - } - p.context = append(p.context, key[len(key)-1]) -} - -// setValue sets the given key to the given value in the current context. -// It will make sure that the key hasn't already been defined, account for -// implicit key groups. -func (p *parser) setValue(key string, value interface{}) { - var tmpHash interface{} - var ok bool - - hash := p.mapping - keyContext := make(Key, 0) - for _, k := range p.context { - keyContext = append(keyContext, k) - if tmpHash, ok = hash[k]; !ok { - p.bug("Context for key '%s' has not been established.", keyContext) - } - switch t := tmpHash.(type) { - case []map[string]interface{}: - // The context is a table of hashes. Pick the most recent table - // defined as the current hash. - hash = t[len(t)-1] - case map[string]interface{}: - hash = t - default: - p.bug("Expected hash to have type 'map[string]interface{}', but "+ - "it has '%T' instead.", tmpHash) - } - } - keyContext = append(keyContext, key) - - if _, ok := hash[key]; ok { - // Typically, if the given key has already been set, then we have - // to raise an error since duplicate keys are disallowed. However, - // it's possible that a key was previously defined implicitly. In this - // case, it is allowed to be redefined concretely. (See the - // `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.) - // - // But we have to make sure to stop marking it as an implicit. (So that - // another redefinition provokes an error.) - // - // Note that since it has already been defined (as a hash), we don't - // want to overwrite it. So our business is done. - if p.isImplicit(keyContext) { - p.removeImplicit(keyContext) - return - } - - // Otherwise, we have a concrete key trying to override a previous - // key, which is *always* wrong. - p.panicf("Key '%s' has already been defined.", keyContext) - } - hash[key] = value -} - -// setType sets the type of a particular value at a given key. -// It should be called immediately AFTER setValue. -// -// Note that if `key` is empty, then the type given will be applied to the -// current context (which is either a table or an array of tables). -func (p *parser) setType(key string, typ tomlType) { - keyContext := make(Key, 0, len(p.context)+1) - for _, k := range p.context { - keyContext = append(keyContext, k) - } - if len(key) > 0 { // allow type setting for hashes - keyContext = append(keyContext, key) - } - p.types[keyContext.String()] = typ -} - -// addImplicit sets the given Key as having been created implicitly. -func (p *parser) addImplicit(key Key) { - p.implicits[key.String()] = true -} - -// removeImplicit stops tagging the given key as having been implicitly -// created. -func (p *parser) removeImplicit(key Key) { - p.implicits[key.String()] = false -} - -// isImplicit returns true if the key group pointed to by the key was created -// implicitly. -func (p *parser) isImplicit(key Key) bool { - return p.implicits[key.String()] -} - -// current returns the full key name of the current context. -func (p *parser) current() string { - if len(p.currentKey) == 0 { - return p.context.String() - } - if len(p.context) == 0 { - return p.currentKey - } - return fmt.Sprintf("%s.%s", p.context, p.currentKey) -} - -func stripFirstNewline(s string) string { - if len(s) == 0 || s[0] != '\n' { - return s - } - return s[1:] -} - -func stripEscapedWhitespace(s string) string { - esc := strings.Split(s, "\\\n") - if len(esc) > 1 { - for i := 1; i < len(esc); i++ { - esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace) - } - } - return strings.Join(esc, "") -} - -func (p *parser) replaceEscapes(str string) string { - var replaced []rune - s := []byte(str) - r := 0 - for r < len(s) { - if s[r] != '\\' { - c, size := utf8.DecodeRune(s[r:]) - r += size - replaced = append(replaced, c) - continue - } - r += 1 - if r >= len(s) { - p.bug("Escape sequence at end of string.") - return "" - } - switch s[r] { - default: - p.bug("Expected valid escape code after \\, but got %q.", s[r]) - return "" - case 'b': - replaced = append(replaced, rune(0x0008)) - r += 1 - case 't': - replaced = append(replaced, rune(0x0009)) - r += 1 - case 'n': - replaced = append(replaced, rune(0x000A)) - r += 1 - case 'f': - replaced = append(replaced, rune(0x000C)) - r += 1 - case 'r': - replaced = append(replaced, rune(0x000D)) - r += 1 - case '"': - replaced = append(replaced, rune(0x0022)) - r += 1 - case '\\': - replaced = append(replaced, rune(0x005C)) - r += 1 - case 'u': - // At this point, we know we have a Unicode escape of the form - // `uXXXX` at [r, r+5). (Because the lexer guarantees this - // for us.) - escaped := p.asciiEscapeToUnicode(s[r+1 : r+5]) - replaced = append(replaced, escaped) - r += 5 - case 'U': - // At this point, we know we have a Unicode escape of the form - // `uXXXX` at [r, r+9). (Because the lexer guarantees this - // for us.) - escaped := p.asciiEscapeToUnicode(s[r+1 : r+9]) - replaced = append(replaced, escaped) - r += 9 - } - } - return string(replaced) -} - -func (p *parser) asciiEscapeToUnicode(bs []byte) rune { - s := string(bs) - hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32) - if err != nil { - p.bug("Could not parse '%s' as a hexadecimal number, but the "+ - "lexer claims it's OK: %s", s, err) - } - if !utf8.ValidRune(rune(hex)) { - p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s) - } - return rune(hex) -} - -func isStringType(ty itemType) bool { - return ty == itemString || ty == itemMultilineString || - ty == itemRawString || ty == itemRawMultilineString -} diff --git a/vendor/github.com/BurntSushi/toml/session.vim b/vendor/github.com/BurntSushi/toml/session.vim deleted file mode 100644 index 562164be..00000000 --- a/vendor/github.com/BurntSushi/toml/session.vim +++ /dev/null @@ -1 +0,0 @@ -au BufWritePost *.go silent!make tags > /dev/null 2>&1 diff --git a/vendor/github.com/BurntSushi/toml/type_check.go b/vendor/github.com/BurntSushi/toml/type_check.go deleted file mode 100644 index c73f8afc..00000000 --- a/vendor/github.com/BurntSushi/toml/type_check.go +++ /dev/null @@ -1,91 +0,0 @@ -package toml - -// tomlType represents any Go type that corresponds to a TOML type. -// While the first draft of the TOML spec has a simplistic type system that -// probably doesn't need this level of sophistication, we seem to be militating -// toward adding real composite types. -type tomlType interface { - typeString() string -} - -// typeEqual accepts any two types and returns true if they are equal. -func typeEqual(t1, t2 tomlType) bool { - if t1 == nil || t2 == nil { - return false - } - return t1.typeString() == t2.typeString() -} - -func typeIsHash(t tomlType) bool { - return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash) -} - -type tomlBaseType string - -func (btype tomlBaseType) typeString() string { - return string(btype) -} - -func (btype tomlBaseType) String() string { - return btype.typeString() -} - -var ( - tomlInteger tomlBaseType = "Integer" - tomlFloat tomlBaseType = "Float" - tomlDatetime tomlBaseType = "Datetime" - tomlString tomlBaseType = "String" - tomlBool tomlBaseType = "Bool" - tomlArray tomlBaseType = "Array" - tomlHash tomlBaseType = "Hash" - tomlArrayHash tomlBaseType = "ArrayHash" -) - -// typeOfPrimitive returns a tomlType of any primitive value in TOML. -// Primitive values are: Integer, Float, Datetime, String and Bool. -// -// Passing a lexer item other than the following will cause a BUG message -// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime. -func (p *parser) typeOfPrimitive(lexItem item) tomlType { - switch lexItem.typ { - case itemInteger: - return tomlInteger - case itemFloat: - return tomlFloat - case itemDatetime: - return tomlDatetime - case itemString: - return tomlString - case itemMultilineString: - return tomlString - case itemRawString: - return tomlString - case itemRawMultilineString: - return tomlString - case itemBool: - return tomlBool - } - p.bug("Cannot infer primitive type of lex item '%s'.", lexItem) - panic("unreachable") -} - -// typeOfArray returns a tomlType for an array given a list of types of its -// values. -// -// In the current spec, if an array is homogeneous, then its type is always -// "Array". If the array is not homogeneous, an error is generated. -func (p *parser) typeOfArray(types []tomlType) tomlType { - // Empty arrays are cool. - if len(types) == 0 { - return tomlArray - } - - theType := types[0] - for _, t := range types[1:] { - if !typeEqual(theType, t) { - p.panicf("Array contains values of type '%s' and '%s', but "+ - "arrays must be homogeneous.", theType, t) - } - } - return tomlArray -} diff --git a/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/BurntSushi/toml/type_fields.go deleted file mode 100644 index 6da608af..00000000 --- a/vendor/github.com/BurntSushi/toml/type_fields.go +++ /dev/null @@ -1,241 +0,0 @@ -package toml - -// Struct field handling is adapted from code in encoding/json: -// -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the Go distribution. - -import ( - "reflect" - "sort" - "sync" -) - -// A field represents a single field found in a struct. -type field struct { - name string // the name of the field (`toml` tag included) - tag bool // whether field has a `toml` tag - index []int // represents the depth of an anonymous field - typ reflect.Type // the type of the field -} - -// byName sorts field by name, breaking ties with depth, -// then breaking ties with "name came from toml tag", then -// breaking ties with index sequence. -type byName []field - -func (x byName) Len() int { return len(x) } - -func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byName) Less(i, j int) bool { - if x[i].name != x[j].name { - return x[i].name < x[j].name - } - if len(x[i].index) != len(x[j].index) { - return len(x[i].index) < len(x[j].index) - } - if x[i].tag != x[j].tag { - return x[i].tag - } - return byIndex(x).Less(i, j) -} - -// byIndex sorts field by index sequence. -type byIndex []field - -func (x byIndex) Len() int { return len(x) } - -func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byIndex) Less(i, j int) bool { - for k, xik := range x[i].index { - if k >= len(x[j].index) { - return false - } - if xik != x[j].index[k] { - return xik < x[j].index[k] - } - } - return len(x[i].index) < len(x[j].index) -} - -// typeFields returns a list of fields that TOML should recognize for the given -// type. The algorithm is breadth-first search over the set of structs to -// include - the top struct and then any reachable anonymous structs. -func typeFields(t reflect.Type) []field { - // Anonymous fields to explore at the current level and the next. - current := []field{} - next := []field{{typ: t}} - - // Count of queued names for current level and the next. - count := map[reflect.Type]int{} - nextCount := map[reflect.Type]int{} - - // Types already visited at an earlier level. - visited := map[reflect.Type]bool{} - - // Fields found. - var fields []field - - for len(next) > 0 { - current, next = next, current[:0] - count, nextCount = nextCount, map[reflect.Type]int{} - - for _, f := range current { - if visited[f.typ] { - continue - } - visited[f.typ] = true - - // Scan f.typ for fields to include. - for i := 0; i < f.typ.NumField(); i++ { - sf := f.typ.Field(i) - if sf.PkgPath != "" && !sf.Anonymous { // unexported - continue - } - name, _ := getOptions(sf.Tag.Get("toml")) - if name == "-" { - continue - } - index := make([]int, len(f.index)+1) - copy(index, f.index) - index[len(f.index)] = i - - ft := sf.Type - if ft.Name() == "" && ft.Kind() == reflect.Ptr { - // Follow pointer. - ft = ft.Elem() - } - - // Record found field and index sequence. - if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { - tagged := name != "" - if name == "" { - name = sf.Name - } - fields = append(fields, field{name, tagged, index, ft}) - if count[f.typ] > 1 { - // If there were multiple instances, add a second, - // so that the annihilation code will see a duplicate. - // It only cares about the distinction between 1 or 2, - // so don't bother generating any more copies. - fields = append(fields, fields[len(fields)-1]) - } - continue - } - - // Record new anonymous struct to explore in next round. - nextCount[ft]++ - if nextCount[ft] == 1 { - f := field{name: ft.Name(), index: index, typ: ft} - next = append(next, f) - } - } - } - } - - sort.Sort(byName(fields)) - - // Delete all fields that are hidden by the Go rules for embedded fields, - // except that fields with TOML tags are promoted. - - // The fields are sorted in primary order of name, secondary order - // of field index length. Loop over names; for each name, delete - // hidden fields by choosing the one dominant field that survives. - out := fields[:0] - for advance, i := 0, 0; i < len(fields); i += advance { - // One iteration per name. - // Find the sequence of fields with the name of this first field. - fi := fields[i] - name := fi.name - for advance = 1; i+advance < len(fields); advance++ { - fj := fields[i+advance] - if fj.name != name { - break - } - } - if advance == 1 { // Only one field with this name - out = append(out, fi) - continue - } - dominant, ok := dominantField(fields[i : i+advance]) - if ok { - out = append(out, dominant) - } - } - - fields = out - sort.Sort(byIndex(fields)) - - return fields -} - -// dominantField looks through the fields, all of which are known to -// have the same name, to find the single field that dominates the -// others using Go's embedding rules, modified by the presence of -// TOML tags. If there are multiple top-level fields, the boolean -// will be false: This condition is an error in Go and we skip all -// the fields. -func dominantField(fields []field) (field, bool) { - // The fields are sorted in increasing index-length order. The winner - // must therefore be one with the shortest index length. Drop all - // longer entries, which is easy: just truncate the slice. - length := len(fields[0].index) - tagged := -1 // Index of first tagged field. - for i, f := range fields { - if len(f.index) > length { - fields = fields[:i] - break - } - if f.tag { - if tagged >= 0 { - // Multiple tagged fields at the same level: conflict. - // Return no field. - return field{}, false - } - tagged = i - } - } - if tagged >= 0 { - return fields[tagged], true - } - // All remaining fields have the same length. If there's more than one, - // we have a conflict (two fields named "X" at the same level) and we - // return no field. - if len(fields) > 1 { - return field{}, false - } - return fields[0], true -} - -var fieldCache struct { - sync.RWMutex - m map[reflect.Type][]field -} - -// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. -func cachedTypeFields(t reflect.Type) []field { - fieldCache.RLock() - f := fieldCache.m[t] - fieldCache.RUnlock() - if f != nil { - return f - } - - // Compute fields without lock. - // Might duplicate effort but won't hold other computations back. - f = typeFields(t) - if f == nil { - f = []field{} - } - - fieldCache.Lock() - if fieldCache.m == nil { - fieldCache.m = map[reflect.Type][]field{} - } - fieldCache.m[t] = f - fieldCache.Unlock() - return f -} diff --git a/vendor/github.com/blevesearch/bleve/CONTRIBUTING.md b/vendor/github.com/blevesearch/bleve/CONTRIBUTING.md new file mode 100644 index 00000000..5ebf3d65 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/CONTRIBUTING.md @@ -0,0 +1,16 @@ +# Contributing to Bleve + +We look forward to your contributions, but ask that you first review these guidelines. + +### Sign the CLA + +As Bleve is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Bleve project does not use this code review tool but it is still used to track acceptance of the contributor license agreements. + +### Submitting a Pull Request + +All types of contributions are welcome, but please keep the following in mind: + +- If you're planning a large change, you should really discuss it in a github issue or on the google group first. This helps avoid duplicate effort and spending time on something that may not be merged. +- Existing tests should continue to pass, new tests for the contribution are nice to have. +- All code should have gone through `go fmt` +- All code should pass `go vet` diff --git a/vendor/github.com/blevesearch/bleve/README.md b/vendor/github.com/blevesearch/bleve/README.md index 4289dc2f..fa11f906 100644 --- a/vendor/github.com/blevesearch/bleve/README.md +++ b/vendor/github.com/blevesearch/bleve/README.md @@ -1,10 +1,14 @@ # ![bleve](docs/bleve.png) bleve -[![Build Status](https://travis-ci.org/blevesearch/bleve.svg?branch=master)](https://travis-ci.org/blevesearch/bleve) [![Coverage Status](https://coveralls.io/repos/blevesearch/bleve/badge.png?branch=master)](https://coveralls.io/r/blevesearch/bleve?branch=master) [![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) [![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Build Status](https://travis-ci.org/blevesearch/bleve.svg?branch=master)](https://travis-ci.org/blevesearch/bleve) [![Coverage Status](https://coveralls.io/repos/blevesearch/bleve/badge.png?branch=master)](https://coveralls.io/r/blevesearch/bleve?branch=master) [![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) +[![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve) +[![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve) +[![Sourcegraph](https://sourcegraph.com/github.com/blevesearch/bleve/-/badge.svg)](https://sourcegraph.com/github.com/blevesearch/bleve?badge) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) modern text indexing in go - [blevesearch.com](http://www.blevesearch.com/) -Try out bleve live by [searching our wiki](http://wikisearch.blevesearch.com/search/). +Try out bleve live by [searching the bleve website](http://www.blevesearch.com/search/?q=bleve). ## Features @@ -16,7 +20,7 @@ Try out bleve live by [searching our wiki](http://wikisearch.blevesearch.com/sea * Term, Phrase, Match, Match Phrase, Prefix * Conjunction, Disjunction, Boolean * Numeric Range, Date Range - * Simple query [syntax](https://github.com/blevesearch/bleve/wiki/Query-String-Query) for human entry + * Simple query [syntax](http://www.blevesearch.com/docs/Query-String-Query/) for human entry * tf-idf Scoring * Search result match highlighting * Supports Aggregating Facets: @@ -30,32 +34,34 @@ Discuss usage and development of bleve in the [google group](https://groups.goog ## Indexing - message := struct{ - Id string - From string - Body string - }{ - Id: "example", - From: "marty.schoch@gmail.com", - Body: "bleve indexing is easy", - } +```go +message := struct{ + Id string + From string + Body string +}{ + Id: "example", + From: "marty.schoch@gmail.com", + Body: "bleve indexing is easy", +} - mapping := bleve.NewIndexMapping() - index, err := bleve.New("example.bleve", mapping) - if err != nil { - panic(err) - } - index.Index(message.Id, message) +mapping := bleve.NewIndexMapping() +index, err := bleve.New("example.bleve", mapping) +if err != nil { + panic(err) +} +index.Index(message.Id, message) +``` ## Querying - index, _ := bleve.Open("example.bleve") - query := bleve.NewQueryStringQuery("bleve") - searchRequest := bleve.NewSearchRequest(query) - searchResult, _ := index.Search(searchRequest) +```go +index, _ := bleve.Open("example.bleve") +query := bleve.NewQueryStringQuery("bleve") +searchRequest := bleve.NewSearchRequest(query) +searchResult, _ := index.Search(searchRequest) +``` ## License Apache License Version 2.0 - - diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go b/vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go new file mode 100644 index 00000000..74ea564e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/analysis/analyzer/standard/standard.go @@ -0,0 +1,52 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package standard + +import ( + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/analysis/lang/en" + "github.com/blevesearch/bleve/analysis/token/lowercase" + "github.com/blevesearch/bleve/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/registry" +) + +const Name = "standard" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopEnFilter, err := cache.TokenFilterNamed(en.StopName) + if err != nil { + return nil, err + } + rv := analysis.Analyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + toLowerFilter, + stopEnFilter, + }, + } + return &rv, nil +} + +func init() { + registry.RegisterAnalyzer(Name, AnalyzerConstructor) +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer/custom_analyzer.go b/vendor/github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer/custom_analyzer.go deleted file mode 100644 index 6fbb9f43..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer/custom_analyzer.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package standard_analyzer - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "custom" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - - var err error - var charFilters []analysis.CharFilter - charFiltersNames, ok := config["char_filters"].([]string) - if ok { - charFilters, err = getCharFilters(charFiltersNames, cache) - if err != nil { - return nil, err - } - } else { - charFiltersNamesInterfaceSlice, ok := config["char_filters"].([]interface{}) - if ok { - charFiltersNames, err := convertInterfaceSliceToStringSlice(charFiltersNamesInterfaceSlice, "char filter") - if err != nil { - return nil, err - } - charFilters, err = getCharFilters(charFiltersNames, cache) - if err != nil { - return nil, err - } - } - } - - tokenizerName, ok := config["tokenizer"].(string) - if !ok { - return nil, fmt.Errorf("must specify tokenizer") - } - - tokenizer, err := cache.TokenizerNamed(tokenizerName) - if err != nil { - return nil, err - } - - var tokenFilters []analysis.TokenFilter - tokenFiltersNames, ok := config["token_filters"].([]string) - if ok { - tokenFilters, err = getTokenFilters(tokenFiltersNames, cache) - if err != nil { - return nil, err - } - } else { - tokenFiltersNamesInterfaceSlice, ok := config["token_filters"].([]interface{}) - if ok { - tokenFiltersNames, err := convertInterfaceSliceToStringSlice(tokenFiltersNamesInterfaceSlice, "token filter") - if err != nil { - return nil, err - } - tokenFilters, err = getTokenFilters(tokenFiltersNames, cache) - if err != nil { - return nil, err - } - } - } - - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - } - if charFilters != nil { - rv.CharFilters = charFilters - } - if tokenFilters != nil { - rv.TokenFilters = tokenFilters - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(Name, AnalyzerConstructor) -} - -func getCharFilters(charFilterNames []string, cache *registry.Cache) ([]analysis.CharFilter, error) { - charFilters := make([]analysis.CharFilter, len(charFilterNames)) - for i, charFilterName := range charFilterNames { - charFilter, err := cache.CharFilterNamed(charFilterName) - if err != nil { - return nil, err - } - charFilters[i] = charFilter - } - - return charFilters, nil -} - -func getTokenFilters(tokenFilterNames []string, cache *registry.Cache) ([]analysis.TokenFilter, error) { - tokenFilters := make([]analysis.TokenFilter, len(tokenFilterNames)) - for i, tokenFilterName := range tokenFilterNames { - tokenFilter, err := cache.TokenFilterNamed(tokenFilterName) - if err != nil { - return nil, err - } - tokenFilters[i] = tokenFilter - } - - return tokenFilters, nil -} - -func convertInterfaceSliceToStringSlice(interfaceSlice []interface{}, objType string) ([]string, error) { - stringSlice := make([]string, len(interfaceSlice)) - for i, interfaceObj := range interfaceSlice { - stringObj, ok := interfaceObj.(string) - if ok { - stringSlice[i] = stringObj - } else { - return nil, fmt.Errorf(objType + " name must be a string") - } - } - - return stringSlice, nil -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzers/detect_lang_analyzer/detect_lang_analyzer.go b/vendor/github.com/blevesearch/bleve/analysis/analyzers/detect_lang_analyzer/detect_lang_analyzer.go deleted file mode 100644 index cb703e6a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/analyzers/detect_lang_analyzer/detect_lang_analyzer.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build cld2 full - -package detect_lang_analyzer - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/cld2" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/single_token" - "github.com/blevesearch/bleve/registry" -) - -const Name = "detect_lang" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - keywordTokenizer, err := cache.TokenizerNamed(single_token.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - detectLangFilter, err := cache.TokenFilterNamed(cld2.Name) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: keywordTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - detectLangFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(Name, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer/keyword_analyzer.go b/vendor/github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer/keyword_analyzer.go deleted file mode 100644 index c8e6b20c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer/keyword_analyzer.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package keyword_analyzer - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizers/single_token" - "github.com/blevesearch/bleve/registry" -) - -const Name = "keyword" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - keywordTokenizer, err := cache.TokenizerNamed(single_token.Name) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: keywordTokenizer, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(Name, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzers/simple_analyzer/simple_analyzer.go b/vendor/github.com/blevesearch/bleve/analysis/analyzers/simple_analyzer/simple_analyzer.go deleted file mode 100644 index 7c7e9e71..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/analyzers/simple_analyzer/simple_analyzer.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package simple_analyzer - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" - "github.com/blevesearch/bleve/registry" -) - -const Name = "simple" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(Name, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer/standard_analyzer.go b/vendor/github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer/standard_analyzer.go deleted file mode 100644 index 0f628885..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer/standard_analyzer.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package standard_analyzer - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/language/en" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" - "github.com/blevesearch/bleve/registry" -) - -const Name = "standard" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopEnFilter, err := cache.TokenFilterNamed(en.StopName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopEnFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(Name, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/ignore/ignore_byte_array_converter.go b/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/ignore/ignore_byte_array_converter.go deleted file mode 100644 index 84695f32..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/ignore/ignore_byte_array_converter.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ignore_byte_array_converter - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -type IgnoreByteArrayConverter struct{} - -func NewIgnoreByteArrayConverter() *IgnoreByteArrayConverter { - return &IgnoreByteArrayConverter{} -} - -func (c *IgnoreByteArrayConverter) Convert(in []byte) (interface{}, error) { - return nil, nil -} - -func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) { - return NewIgnoreByteArrayConverter(), nil -} - -func init() { - registry.RegisterByteArrayConverter("ignore", Constructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/json/json_byte_array_converter.go b/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/json/json_byte_array_converter.go deleted file mode 100644 index 138cf982..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/json/json_byte_array_converter.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package json_byte_array_converter - -import ( - "encoding/json" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -type JSONByteArrayConverter struct{} - -func NewJSONByteArrayConverter() *JSONByteArrayConverter { - return &JSONByteArrayConverter{} -} - -func (c *JSONByteArrayConverter) Convert(in []byte) (interface{}, error) { - var rv map[string]interface{} - err := json.Unmarshal(in, &rv) - if err != nil { - return nil, err - } - return rv, nil -} - -func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) { - return NewJSONByteArrayConverter(), nil -} - -func init() { - registry.RegisterByteArrayConverter("json", Constructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/string/string_byte_array_conveter.go b/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/string/string_byte_array_conveter.go deleted file mode 100644 index 6ab5f658..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/byte_array_converters/string/string_byte_array_conveter.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package string_byte_array_converter - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -type StringByteArrayConverter struct{} - -func NewStringByteArrayConverter() *StringByteArrayConverter { - return &StringByteArrayConverter{} -} - -func (c *StringByteArrayConverter) Convert(in []byte) (interface{}, error) { - return string(in), nil -} - -func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) { - return NewStringByteArrayConverter(), nil -} - -func init() { - registry.RegisterByteArrayConverter("string", Constructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/char_filters/html_char_filter/html_char_filter.go b/vendor/github.com/blevesearch/bleve/analysis/char_filters/html_char_filter/html_char_filter.go deleted file mode 100644 index a633004c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/char_filters/html_char_filter/html_char_filter.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package html_char_filter - -import ( - "regexp" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter" - "github.com/blevesearch/bleve/registry" -) - -const Name = "html" - -var htmlCharFilterRegexp = regexp.MustCompile(`\s]+))?)+\s*|\s*)/?>`) - -func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { - replaceBytes := []byte(" ") - return regexp_char_filter.NewRegexpCharFilter(htmlCharFilterRegexp, replaceBytes), nil -} - -func init() { - registry.RegisterCharFilter(Name, CharFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter.go b/vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter.go deleted file mode 100644 index b0bfa14d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package regexp_char_filter - -import ( - "bytes" - "fmt" - "regexp" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "regexp" - -type RegexpCharFilter struct { - r *regexp.Regexp - replacement []byte -} - -func NewRegexpCharFilter(r *regexp.Regexp, replacement []byte) *RegexpCharFilter { - return &RegexpCharFilter{ - r: r, - replacement: replacement, - } -} - -func (s *RegexpCharFilter) Filter(input []byte) []byte { - return s.r.ReplaceAllFunc(input, func(in []byte) []byte { return bytes.Repeat(s.replacement, len(in)) }) -} - -func RegexpCharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { - regexpStr, ok := config["regexp"].(string) - if !ok { - return nil, fmt.Errorf("must specify regexp") - } - r, err := regexp.Compile(regexpStr) - if err != nil { - return nil, fmt.Errorf("unable to build regexp char filter: %v", err) - } - replaceBytes := []byte(" ") - replaceStr, ok := config["replace"].(string) - if ok { - replaceBytes = []byte(replaceStr) - } - return NewRegexpCharFilter(r, replaceBytes), nil -} - -func init() { - registry.RegisterCharFilter(Name, RegexpCharFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter_test.go deleted file mode 100644 index d51ed78c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter/regexp_char_filter_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package regexp_char_filter - -import ( - "reflect" - "regexp" - "testing" -) - -func TestRegexpCharFilter(t *testing.T) { - - htmlTagPattern := `\s]+))?)+\s*|\s*)/?>` - htmlRegex := regexp.MustCompile(htmlTagPattern) - - tests := []struct { - input []byte - output []byte - }{ - { - input: []byte(` - - - -

My First Heading

- -

My first paragraph.

- - -`), - output: []byte(` - - - - My First Heading - - My first paragraph. - - - `), - }, - } - - for _, test := range tests { - filter := NewRegexpCharFilter(htmlRegex, []byte{' '}) - output := filter.Filter(test.input) - if !reflect.DeepEqual(output, test.output) { - t.Errorf("Expected:\n`%s`\ngot:\n`%s`\nfor:\n`%s`\n", string(test.output), string(output), string(test.input)) - } - } -} - -func TestZeroWidthNonJoinerCharFilter(t *testing.T) { - - zeroWidthNonJoinerPattern := `\x{200C}` - zeroWidthNonJoinerRegex := regexp.MustCompile(zeroWidthNonJoinerPattern) - - tests := []struct { - input []byte - output []byte - }{ - { - input: []byte("water\u200Cunder\u200Cthe\u200Cbridge"), - output: []byte("water under the bridge"), - }, - } - - for _, test := range tests { - filter := NewRegexpCharFilter(zeroWidthNonJoinerRegex, []byte{' '}) - output := filter.Filter(test.input) - if !reflect.DeepEqual(output, test.output) { - t.Errorf("Expected:\n`%s`\ngot:\n`%s`\nfor:\n`%s`\n", string(test.output), string(output), string(test.input)) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/char_filters/zero_width_non_joiner/zero_width_non_joiner_char_filter.go b/vendor/github.com/blevesearch/bleve/analysis/char_filters/zero_width_non_joiner/zero_width_non_joiner_char_filter.go deleted file mode 100644 index cd507e5d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/char_filters/zero_width_non_joiner/zero_width_non_joiner_char_filter.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package zero_width_non_joiner - -import ( - "regexp" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter" - "github.com/blevesearch/bleve/registry" -) - -const Name = "zero_width_spaces" - -var zeroWidthNonJoinerRegexp = regexp.MustCompile(`\x{200C}`) - -func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { - replaceBytes := []byte(" ") - return regexp_char_filter.NewRegexpCharFilter(zeroWidthNonJoinerRegexp, replaceBytes), nil -} - -func init() { - registry.RegisterCharFilter(Name, CharFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go b/vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go new file mode 100644 index 00000000..cd549f55 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/analysis/datetime/flexible/flexible.go @@ -0,0 +1,64 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flexible + +import ( + "fmt" + "time" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/registry" +) + +const Name = "flexiblego" + +type DateTimeParser struct { + layouts []string +} + +func New(layouts []string) *DateTimeParser { + return &DateTimeParser{ + layouts: layouts, + } +} + +func (p *DateTimeParser) ParseDateTime(input string) (time.Time, error) { + for _, layout := range p.layouts { + rv, err := time.Parse(layout, input) + if err == nil { + return rv, nil + } + } + return time.Time{}, analysis.ErrInvalidDateTime +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + layouts, ok := config["layouts"].([]interface{}) + if !ok { + return nil, fmt.Errorf("must specify layouts") + } + var layoutStrs []string + for _, layout := range layouts { + layoutStr, ok := layout.(string) + if ok { + layoutStrs = append(layoutStrs, layoutStr) + } + } + return New(layoutStrs), nil +} + +func init() { + registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go b/vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go new file mode 100644 index 00000000..4b98de66 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/analysis/datetime/optional/optional.go @@ -0,0 +1,45 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package optional + +import ( + "time" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/analysis/datetime/flexible" + "github.com/blevesearch/bleve/registry" +) + +const Name = "dateTimeOptional" + +const rfc3339NoTimezone = "2006-01-02T15:04:05" +const rfc3339NoTimezoneNoT = "2006-01-02 15:04:05" +const rfc3339NoTime = "2006-01-02" + +var layouts = []string{ + time.RFC3339Nano, + time.RFC3339, + rfc3339NoTimezone, + rfc3339NoTimezoneNoT, + rfc3339NoTime, +} + +func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { + return flexible.New(layouts), nil +} + +func init() { + registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional/datetime_optional.go b/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional/datetime_optional.go deleted file mode 100644 index d812c4c0..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional/datetime_optional.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package html_char_filter - -import ( - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go" - "github.com/blevesearch/bleve/registry" -) - -const Name = "dateTimeOptional" - -const rfc3339NoTimezone = "2006-01-02T15:04:05" -const rfc3339NoTimezoneNoT = "2006-01-02 15:04:05" -const rfc3339NoTime = "2006-01-02" - -var layouts = []string{ - time.RFC3339Nano, - time.RFC3339, - rfc3339NoTimezone, - rfc3339NoTimezoneNoT, - rfc3339NoTime, -} - -func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { - return flexible_go.NewFlexibleGoDateTimeParser(layouts), nil -} - -func init() { - registry.RegisterDateTimeParser(Name, DateTimeParserConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go.go b/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go.go deleted file mode 100644 index fa613d6e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package flexible_go - -import ( - "fmt" - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "flexiblego" - -type FlexibleGoDateTimeParser struct { - layouts []string -} - -func NewFlexibleGoDateTimeParser(layouts []string) *FlexibleGoDateTimeParser { - return &FlexibleGoDateTimeParser{ - layouts: layouts, - } -} - -func (p *FlexibleGoDateTimeParser) ParseDateTime(input string) (time.Time, error) { - for _, layout := range p.layouts { - rv, err := time.Parse(layout, input) - if err == nil { - return rv, nil - } - } - return time.Time{}, analysis.ErrInvalidDateTime -} - -func FlexibleGoDateTimeParserConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.DateTimeParser, error) { - layouts, ok := config["layouts"].([]interface{}) - if !ok { - return nil, fmt.Errorf("must specify layouts") - } - layoutStrs := make([]string, 0) - for _, layout := range layouts { - layoutStr, ok := layout.(string) - if ok { - layoutStrs = append(layoutStrs, layoutStr) - } - } - return NewFlexibleGoDateTimeParser(layoutStrs), nil -} - -func init() { - registry.RegisterDateTimeParser(Name, FlexibleGoDateTimeParserConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go_test.go b/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go_test.go deleted file mode 100644 index 37c14e95..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go/flexible_go_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package flexible_go - -import ( - "reflect" - "testing" - "time" - - "github.com/blevesearch/bleve/analysis" -) - -func TestFlexibleDateTimeParser(t *testing.T) { - testLocation := time.FixedZone("", -8*60*60) - - tests := []struct { - input string - expectedTime time.Time - expectedError error - }{ - { - input: "2014-08-03", - expectedTime: time.Date(2014, 8, 3, 0, 0, 0, 0, time.UTC), - expectedError: nil, - }, - { - input: "2014-08-03T15:59:30", - expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 0, time.UTC), - expectedError: nil, - }, - { - input: "2014-08-03 15:59:30", - expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 0, time.UTC), - expectedError: nil, - }, - { - input: "2014-08-03T15:59:30-08:00", - expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 0, testLocation), - expectedError: nil, - }, - { - input: "2014-08-03T15:59:30.999999999-08:00", - expectedTime: time.Date(2014, 8, 3, 15, 59, 30, 999999999, testLocation), - expectedError: nil, - }, - { - input: "not a date time", - expectedTime: time.Time{}, - expectedError: analysis.ErrInvalidDateTime, - }, - } - - rfc3339NoTimezone := "2006-01-02T15:04:05" - rfc3339NoTimezoneNoT := "2006-01-02 15:04:05" - rfc3339NoTime := "2006-01-02" - - dateOptionalTimeParser := NewFlexibleGoDateTimeParser( - []string{ - time.RFC3339Nano, - time.RFC3339, - rfc3339NoTimezone, - rfc3339NoTimezoneNoT, - rfc3339NoTime, - }) - - for _, test := range tests { - actualTime, actualErr := dateOptionalTimeParser.ParseDateTime(test.input) - if actualErr != test.expectedError { - t.Errorf("expected error %#v, got %#v", test.expectedError, actualErr) - continue - } - if !reflect.DeepEqual(actualTime, test.expectedTime) { - t.Errorf("expected time %#v, got %#v", test.expectedTime, actualTime) - t.Errorf("expected location %#v,\n got %#v", test.expectedTime.Location(), actualTime.Location()) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/freq.go b/vendor/github.com/blevesearch/bleve/analysis/freq.go index 1cf96414..e1ca2cd6 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/freq.go +++ b/vendor/github.com/blevesearch/bleve/analysis/freq.go @@ -1,88 +1,111 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package analysis +// TokenLocation represents one occurrence of a term at a particular location in +// a field. Start, End and Position have the same meaning as in analysis.Token. +// Field and ArrayPositions identify the field value in the source document. +// See document.Field for details. type TokenLocation struct { - Field string - Start int - End int - Position int + Field string + ArrayPositions []uint64 + Start int + End int + Position int } +// TokenFreq represents all the occurrences of a term in all fields of a +// document. type TokenFreq struct { Term []byte Locations []*TokenLocation + frequency int } -type TokenFrequencies []*TokenFreq +func (tf *TokenFreq) Frequency() int { + return tf.frequency +} -func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) TokenFrequencies { - // put existing tokens into a map - index := make(map[string]*TokenFreq) - for _, tf := range tfs { - index[string(tf.Term)] = tf - } +// TokenFrequencies maps document terms to their combined frequencies from all +// fields. +type TokenFrequencies map[string]*TokenFreq + +func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) { // walk the new token frequencies - for _, tf := range other { + for tfk, tf := range other { // set the remoteField value in incoming token freqs for _, l := range tf.Locations { l.Field = remoteField } - existingTf, exists := index[string(tf.Term)] + existingTf, exists := tfs[tfk] if exists { existingTf.Locations = append(existingTf.Locations, tf.Locations...) + existingTf.frequency = existingTf.frequency + tf.frequency } else { - index[string(tf.Term)] = tf + tfs[tfk] = &TokenFreq{ + Term: tf.Term, + frequency: tf.frequency, + Locations: make([]*TokenLocation, len(tf.Locations)), + } + copy(tfs[tfk].Locations, tf.Locations) } } - // flatten map back to array - rv := make(TokenFrequencies, len(index)) - i := 0 - for _, tf := range index { - rv[i] = tf - i++ - } - return rv } -func TokenFrequency(tokens TokenStream) TokenFrequencies { - index := make(map[string]*TokenFreq) +func TokenFrequency(tokens TokenStream, arrayPositions []uint64, includeTermVectors bool) TokenFrequencies { + rv := make(map[string]*TokenFreq, len(tokens)) - for _, token := range tokens { - curr, ok := index[string(token.Term)] - if ok { - curr.Locations = append(curr.Locations, &TokenLocation{ - Start: token.Start, - End: token.End, - Position: token.Position, - }) - } else { - index[string(token.Term)] = &TokenFreq{ - Term: token.Term, - Locations: []*TokenLocation{ - &TokenLocation{ - Start: token.Start, - End: token.End, - Position: token.Position, - }, - }, + if includeTermVectors { + tls := make([]TokenLocation, len(tokens)) + tlNext := 0 + + for _, token := range tokens { + tls[tlNext] = TokenLocation{ + ArrayPositions: arrayPositions, + Start: token.Start, + End: token.End, + Position: token.Position, + } + + curr, ok := rv[string(token.Term)] + if ok { + curr.Locations = append(curr.Locations, &tls[tlNext]) + curr.frequency++ + } else { + rv[string(token.Term)] = &TokenFreq{ + Term: token.Term, + Locations: []*TokenLocation{&tls[tlNext]}, + frequency: 1, + } + } + + tlNext++ + } + } else { + for _, token := range tokens { + curr, exists := rv[string(token.Term)] + if exists { + curr.frequency++ + } else { + rv[string(token.Term)] = &TokenFreq{ + Term: token.Term, + frequency: 1, + } } } } - rv := make(TokenFrequencies, len(index)) - i := 0 - for _, tf := range index { - rv[i] = tf - i++ - } - return rv } diff --git a/vendor/github.com/blevesearch/bleve/analysis/freq_test.go b/vendor/github.com/blevesearch/bleve/analysis/freq_test.go deleted file mode 100644 index 02c945ba..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/freq_test.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package analysis - -import ( - "reflect" - "testing" -) - -func TestTokenFrequency(t *testing.T) { - tokens := TokenStream{ - &Token{ - Term: []byte("water"), - Position: 1, - Start: 0, - End: 5, - }, - &Token{ - Term: []byte("water"), - Position: 2, - Start: 6, - End: 11, - }, - } - expectedResult := TokenFrequencies{ - &TokenFreq{ - Term: []byte("water"), - Locations: []*TokenLocation{ - &TokenLocation{ - Position: 1, - Start: 0, - End: 5, - }, - &TokenLocation{ - Position: 2, - Start: 6, - End: 11, - }, - }, - }, - } - result := TokenFrequency(tokens) - if !reflect.DeepEqual(result, expectedResult) { - t.Errorf("expected %#v, got %#v", expectedResult, result) - } -} - -func TestTokenFrequenciesMergeAll(t *testing.T) { - tf1 := TokenFrequencies{ - &TokenFreq{ - Term: []byte("water"), - Locations: []*TokenLocation{ - &TokenLocation{ - Position: 1, - Start: 0, - End: 5, - }, - &TokenLocation{ - Position: 2, - Start: 6, - End: 11, - }, - }, - }, - } - tf2 := TokenFrequencies{ - &TokenFreq{ - Term: []byte("water"), - Locations: []*TokenLocation{ - &TokenLocation{ - Position: 1, - Start: 0, - End: 5, - }, - &TokenLocation{ - Position: 2, - Start: 6, - End: 11, - }, - }, - }, - } - expectedResult := TokenFrequencies{ - &TokenFreq{ - Term: []byte("water"), - Locations: []*TokenLocation{ - &TokenLocation{ - Position: 1, - Start: 0, - End: 5, - }, - &TokenLocation{ - Position: 2, - Start: 6, - End: 11, - }, - &TokenLocation{ - Field: "tf2", - Position: 1, - Start: 0, - End: 5, - }, - &TokenLocation{ - Field: "tf2", - Position: 2, - Start: 6, - End: 11, - }, - }, - }, - } - tf1.MergeAll("tf2", tf2) - if !reflect.DeepEqual(tf1, expectedResult) { - t.Errorf("expected %#v, got %#v", expectedResult, tf1) - } -} - -func TestTokenFrequenciesMergeAllLeftEmpty(t *testing.T) { - tf1 := TokenFrequencies{} - tf2 := TokenFrequencies{ - &TokenFreq{ - Term: []byte("water"), - Locations: []*TokenLocation{ - &TokenLocation{ - Position: 1, - Start: 0, - End: 5, - }, - &TokenLocation{ - Position: 2, - Start: 6, - End: 11, - }, - }, - }, - } - expectedResult := TokenFrequencies{ - &TokenFreq{ - Term: []byte("water"), - Locations: []*TokenLocation{ - &TokenLocation{ - Field: "tf2", - Position: 1, - Start: 0, - End: 5, - }, - &TokenLocation{ - Field: "tf2", - Position: 2, - Start: 6, - End: 11, - }, - }, - }, - } - result := tf1.MergeAll("tf2", tf2) - if !reflect.DeepEqual(result, expectedResult) { - t.Errorf("expected %#v, got %#v", expectedResult, result) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go b/vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go new file mode 100644 index 00000000..8402785f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/analysis/lang/en/analyzer_en.go @@ -0,0 +1,70 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package en implements an analyzer with reasonable defaults for processing +// English text. +// +// It strips possessive suffixes ('s), transforms tokens to lower case, +// removes stopwords from a built-in list, and applies porter stemming. +// +// The built-in stopwords list is defined in EnglishStopWords. +package en + +import ( + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/registry" + + "github.com/blevesearch/bleve/analysis/token/lowercase" + "github.com/blevesearch/bleve/analysis/token/porter" + "github.com/blevesearch/bleve/analysis/tokenizer/unicode" +) + +const AnalyzerName = "en" + +func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { + tokenizer, err := cache.TokenizerNamed(unicode.Name) + if err != nil { + return nil, err + } + possEnFilter, err := cache.TokenFilterNamed(PossessiveName) + if err != nil { + return nil, err + } + toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name) + if err != nil { + return nil, err + } + stopEnFilter, err := cache.TokenFilterNamed(StopName) + if err != nil { + return nil, err + } + stemmerEnFilter, err := cache.TokenFilterNamed(porter.Name) + if err != nil { + return nil, err + } + rv := analysis.Analyzer{ + Tokenizer: tokenizer, + TokenFilters: []analysis.TokenFilter{ + possEnFilter, + toLowerFilter, + stopEnFilter, + stemmerEnFilter, + }, + } + return &rv, nil +} + +func init() { + registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go b/vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go new file mode 100644 index 00000000..2c06efd6 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/analysis/lang/en/possessive_filter_en.go @@ -0,0 +1,67 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "unicode/utf8" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/registry" +) + +// PossessiveName is the name PossessiveFilter is registered as +// in the bleve registry. +const PossessiveName = "possessive_en" + +const rightSingleQuotationMark = '’' +const apostrophe = '\'' +const fullWidthApostrophe = '＇' + +const apostropheChars = rightSingleQuotationMark + apostrophe + fullWidthApostrophe + +// PossessiveFilter implements a TokenFilter which +// strips the English possessive suffix ('s) from tokens. +// It handle a variety of apostrophe types, is case-insensitive +// and doesn't distinguish between possessive and contraction. +// (ie "She's So Rad" becomes "She So Rad") +type PossessiveFilter struct { +} + +func NewPossessiveFilter() *PossessiveFilter { + return &PossessiveFilter{} +} + +func (s *PossessiveFilter) Filter(input analysis.TokenStream) analysis.TokenStream { + for _, token := range input { + lastRune, lastRuneSize := utf8.DecodeLastRune(token.Term) + if lastRune == 's' || lastRune == 'S' { + nextLastRune, nextLastRuneSize := utf8.DecodeLastRune(token.Term[:len(token.Term)-lastRuneSize]) + if nextLastRune == rightSingleQuotationMark || + nextLastRune == apostrophe || + nextLastRune == fullWidthApostrophe { + token.Term = token.Term[:len(token.Term)-lastRuneSize-nextLastRuneSize] + } + } + } + return input +} + +func PossessiveFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + return NewPossessiveFilter(), nil +} + +func init() { + registry.RegisterTokenFilter(PossessiveName, PossessiveFilterConstructor) +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go b/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go new file mode 100644 index 00000000..bfdb2c97 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_filter_en.go @@ -0,0 +1,33 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package en + +import ( + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/analysis/token/stop" + "github.com/blevesearch/bleve/registry" +) + +func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { + tokenMap, err := cache.TokenMapNamed(StopName) + if err != nil { + return nil, err + } + return stop.NewStopTokensFilter(tokenMap), nil +} + +func init() { + registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/stop_words_en.go b/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go similarity index 98% rename from vendor/github.com/blevesearch/bleve/analysis/language/en/stop_words_en.go rename to vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go index e19c6178..6423cf2c 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/stop_words_en.go +++ b/vendor/github.com/blevesearch/bleve/analysis/lang/en/stop_words_en.go @@ -7,10 +7,11 @@ import ( const StopName = "stop_en" +// EnglishStopWords is the built-in list of stopwords used by the "stop_en" TokenFilter. +// // this content was obtained from: // lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ // ` was changed to ' to allow for literal string - var EnglishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/english/stop.txt | This file is distributed under the BSD License. | See http://snowball.tartarus.org/license.php diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar.go deleted file mode 100644 index 1d7149af..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ar - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" -) - -const AnalyzerName = "ar" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKC) - stopArFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - normalizeArFilter, err := cache.TokenFilterNamed(NormalizeName) - if err != nil { - return nil, err - } - stemmerArFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - normalizeFilter, - stopArFilter, - normalizeArFilter, - stemmerArFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar_test.go deleted file mode 100644 index ecade6b1..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/analyzer_ar_test.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ar - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestArabicAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - input: []byte("كبير"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كبير"), - Position: 1, - Start: 0, - End: 8, - }, - }, - }, - // feminine marker - { - input: []byte("كبيرة"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كبير"), - Position: 1, - Start: 0, - End: 10, - }, - }, - }, - { - input: []byte("مشروب"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("مشروب"), - Position: 1, - Start: 0, - End: 10, - }, - }, - }, - // plural -at - { - input: []byte("مشروبات"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("مشروب"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - // plural -in - { - input: []byte("أمريكيين"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("امريك"), - Position: 1, - Start: 0, - End: 16, - }, - }, - }, - // singular with bare alif - { - input: []byte("امريكي"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("امريك"), - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - { - input: []byte("كتاب"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كتاب"), - Position: 1, - Start: 0, - End: 8, - }, - }, - }, - // definite article - { - input: []byte("الكتاب"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كتاب"), - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - { - input: []byte("ما ملكت أيمانكم"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ملكت"), - Position: 2, - Start: 5, - End: 13, - }, - &analysis.Token{ - Term: []byte("ايمانكم"), - Position: 3, - Start: 14, - End: 28, - }, - }, - }, - // stopwords - { - input: []byte("الذين ملكت أيمانكم"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ملكت"), - Position: 2, - Start: 11, - End: 19, - }, - &analysis.Token{ - Term: []byte("ايمانكم"), - Position: 3, - Start: 20, - End: 34, - }, - }, - }, - // presentation form normalization - { - input: []byte("ﺍﻟﺴﻼﻢ"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سلام"), - Position: 1, - Start: 0, - End: 15, - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize.go deleted file mode 100644 index abf86018..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ar - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const NormalizeName = "normalize_ar" - -const ( - Alef = '\u0627' - AlefMadda = '\u0622' - AlefHamzaAbove = '\u0623' - AlefHamzaBelow = '\u0625' - Yeh = '\u064A' - DotlessYeh = '\u0649' - TehMarbuta = '\u0629' - Heh = '\u0647' - Tatweel = '\u0640' - Fathatan = '\u064B' - Dammatan = '\u064C' - Kasratan = '\u064D' - Fatha = '\u064E' - Damma = '\u064F' - Kasra = '\u0650' - Shadda = '\u0651' - Sukun = '\u0652' -) - -type ArabicNormalizeFilter struct { -} - -func NewArabicNormalizeFilter() *ArabicNormalizeFilter { - return &ArabicNormalizeFilter{} -} - -func (s *ArabicNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - term := normalize(token.Term) - token.Term = term - } - return input -} - -func normalize(input []byte) []byte { - runes := bytes.Runes(input) - for i := 0; i < len(runes); i++ { - switch runes[i] { - case AlefMadda, AlefHamzaAbove, AlefHamzaBelow: - runes[i] = Alef - case DotlessYeh: - runes[i] = Yeh - case TehMarbuta: - runes[i] = Heh - case Tatweel, Kasratan, Dammatan, Fathatan, Fatha, Damma, Kasra, Shadda, Sukun: - runes = analysis.DeleteRune(runes, i) - i-- - } - } - return analysis.BuildTermFromRunes(runes) -} - -func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewArabicNormalizeFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize_test.go deleted file mode 100644 index d8545460..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/arabic_normalize_test.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ar - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestArabicNormalizeFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // AlifMadda - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("آجن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("اجن"), - }, - }, - }, - // AlifHamzaAbove - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("أحمد"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("احمد"), - }, - }, - }, - // AlifHamzaBelow - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("إعاذ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("اعاذ"), - }, - }, - }, - // AlifMaksura - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("بنى"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("بني"), - }, - }, - }, - // TehMarbuta - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("فاطمة"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("فاطمه"), - }, - }, - }, - // Tatweel - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("روبرـــــت"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("روبرت"), - }, - }, - }, - // Fatha - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("مَبنا"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("مبنا"), - }, - }, - }, - // Kasra - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("علِي"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("علي"), - }, - }, - }, - // Damma - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("بُوات"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("بوات"), - }, - }, - }, - // Fathatan - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ولداً"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ولدا"), - }, - }, - }, - // Kasratan - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ولدٍ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ولد"), - }, - }, - }, - // Dammatan - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ولدٌ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ولد"), - }, - }, - }, - // Sukun - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("نلْسون"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("نلسون"), - }, - }, - }, - // Shaddah - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هتميّ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هتمي"), - }, - }, - }, - // empty - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - arabicNormalizeFilter := NewArabicNormalizeFilter() - for _, test := range tests { - actual := arabicNormalizeFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar.go deleted file mode 100644 index c76757b4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ar - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_ar" - -// These were obtained from org.apache.lucene.analysis.ar.ArabicStemmer -var prefixes = [][]rune{ - []rune("ال"), - []rune("وال"), - []rune("بال"), - []rune("كال"), - []rune("فال"), - []rune("لل"), - []rune("و"), -} -var suffixes = [][]rune{ - []rune("ها"), - []rune("ان"), - []rune("ات"), - []rune("ون"), - []rune("ين"), - []rune("يه"), - []rune("ية"), - []rune("ه"), - []rune("ة"), - []rune("ي"), -} - -type ArabicStemmerFilter struct{} - -func NewArabicStemmerFilter() *ArabicStemmerFilter { - return &ArabicStemmerFilter{} -} - -func (s *ArabicStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - term := stem(token.Term) - token.Term = term - } - return input -} - -func canStemPrefix(input, prefix []rune) bool { - // Wa- prefix requires at least 3 characters. - if len(prefix) == 1 && len(input) < 4 { - return false - } - // Other prefixes require only 2. - if len(input)-len(prefix) < 2 { - return false - } - for i := range prefix { - if prefix[i] != input[i] { - return false - } - } - return true -} - -func canStemSuffix(input, suffix []rune) bool { - // All suffixes require at least 2 characters after stemming. - if len(input)-len(suffix) < 2 { - return false - } - stemEnd := len(input) - len(suffix) - for i := range suffix { - if suffix[i] != input[stemEnd+i] { - return false - } - } - return true -} - -func stem(input []byte) []byte { - runes := bytes.Runes(input) - // Strip a single prefix. - for _, p := range prefixes { - if canStemPrefix(runes, p) { - runes = runes[len(p):] - break - } - } - // Strip off multiple suffixes, in their order in the suffixes array. - for _, s := range suffixes { - if canStemSuffix(runes, s) { - runes = runes[:len(runes)-len(s)] - } - } - return analysis.BuildTermFromRunes(runes) -} - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewArabicStemmerFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar_test.go deleted file mode 100644 index 62ed6ef1..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stemmer_ar_test.go +++ /dev/null @@ -1,392 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ar - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestArabicStemmerFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // AlPrefix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("الحسن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("حسن"), - }, - }, - }, - // WalPrefix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("والحسن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("حسن"), - }, - }, - }, - // BalPrefix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("بالحسن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("حسن"), - }, - }, - }, - // KalPrefix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كالحسن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("حسن"), - }, - }, - }, - // FalPrefix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("فالحسن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("حسن"), - }, - }, - }, - // LlPrefix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("للاخر"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("اخر"), - }, - }, - }, - // WaPrefix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("وحسن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("حسن"), - }, - }, - }, - // AhSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("زوجها"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("زوج"), - }, - }, - }, - // AnSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدان"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // AtSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدات"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // WnSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدون"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // YnSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدين"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // YhSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهديه"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // YpSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدية"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // HSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهده"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // PSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدة"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // YSuffix - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدي"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // ComboPrefSuf - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("وساهدون"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // ComboSuf - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهدهات"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ساهد"), - }, - }, - }, - // ShouldntStem - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("الو"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("الو"), - }, - }, - }, - // NonArabic - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("English"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("English"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سلام"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سلام"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("السلام"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سلام"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سلامة"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سلام"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("السلامة"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سلام"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("الوصل"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("وصل"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("والصل"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("صل"), - }, - }, - }, - // Empty - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - arabicStemmerFilter := NewArabicStemmerFilter() - for _, test := range tests { - actual := arabicStemmerFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_filter_ar.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_filter_ar.go deleted file mode 100644 index dd40c31e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_filter_ar.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ar - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_words_ar.go b/vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_words_ar.go deleted file mode 100644 index 09e9a599..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ar/stop_words_ar.go +++ /dev/null @@ -1,149 +0,0 @@ -package ar - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_ar" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis -// ` was changed to ' to allow for literal string - -var ArabicStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -# Cleaned on October 11, 2009 (not normalized, so use before normalization) -# This means that when modifying this list, you might need to add some -# redundant entries, for example containing forms with both أ and ا -من -ومن -منها -منه -في -وفي -فيها -فيه -و -ف -ثم -او -أو -ب -بها -به -ا -أ -اى -اي -أي -أى -لا -ولا -الا -ألا -إلا -لكن -ما -وما -كما -فما -عن -مع -اذا -إذا -ان -أن -إن -انها -أنها -إنها -انه -أنه -إنه -بان -بأن -فان -فأن -وان -وأن -وإن -التى -التي -الذى -الذي -الذين -الى -الي -إلى -إلي -على -عليها -عليه -اما -أما -إما -ايضا -أيضا -كل -وكل -لم -ولم -لن -ولن -هى -هي -هو -وهى -وهي -وهو -فهى -فهي -فهو -انت -أنت -لك -لها -له -هذه -هذا -تلك -ذلك -هناك -كانت -كان -يكون -تكون -وكانت -وكان -غير -بعض -قد -نحو -بين -بينما -منذ -ضمن -حيث -الان -الآن -خلال -بعد -قبل -حتى -عند -عندما -لدى -جميع -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(ArabicStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_filter_bg.go b/vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_filter_bg.go deleted file mode 100644 index bf30f617..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_filter_bg.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bg - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_words_bg.go b/vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_words_bg.go deleted file mode 100644 index a063ed9a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/bg/stop_words_bg.go +++ /dev/null @@ -1,217 +0,0 @@ -package bg - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_bg" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var BulgarianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -а -аз -ако -ала -бе -без -беше -би -бил -била -били -било -близо -бъдат -бъде -бяха -в -вас -ваш -ваша -вероятно -вече -взема -ви -вие -винаги -все -всеки -всички -всичко -всяка -във -въпреки -върху -г -ги -главно -го -д -да -дали -до -докато -докога -дори -досега -доста -е -едва -един -ето -за -зад -заедно -заради -засега -затова -защо -защото -и -из -или -им -има -имат -иска -й -каза -как -каква -какво -както -какъв -като -кога -когато -което -които -кой -който -колко -която -къде -където -към -ли -м -ме -между -мен -ми -мнозина -мога -могат -може -моля -момента -му -н -на -над -назад -най -направи -напред -например -нас -не -него -нея -ни -ние -никой -нито -но -някои -някой -няма -обаче -около -освен -особено -от -отгоре -отново -още -пак -по -повече -повечето -под -поне -поради -после -почти -прави -пред -преди -през -при -пък -първо -с -са -само -се -сега -си -скоро -след -сме -според -сред -срещу -сте -съм -със -също -т -тази -така -такива -такъв -там -твой -те -тези -ти -тн -то -това -тогава -този -той -толкова -точно -трябва -тук -тъй -тя -тях -у -харесва -ч -че -често -чрез -ще -щом -я -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(BulgarianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ca/articles_ca.go b/vendor/github.com/blevesearch/bleve/analysis/language/ca/articles_ca.go deleted file mode 100644 index 7d52cb16..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ca/articles_ca.go +++ /dev/null @@ -1,30 +0,0 @@ -package ca - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const ArticlesName = "articles_ca" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis - -var CatalanArticles = []byte(` -d -l -m -n -s -t -`) - -func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(CatalanArticles) - return rv, err -} - -func init() { - registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca.go b/vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca.go deleted file mode 100644 index b4b4be9b..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ca - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/elision_filter" - "github.com/blevesearch/bleve/registry" -) - -const ElisionName = "elision_ca" - -func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) - if err != nil { - return nil, fmt.Errorf("error building elision filter: %v", err) - } - return elision_filter.NewElisionFilter(articlesTokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca_test.go deleted file mode 100644 index 6cab8697..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ca/elision_ca_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ca - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestFrenchElision(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("l'Institut"), - }, - &analysis.Token{ - Term: []byte("d'Estudis"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Institut"), - }, - &analysis.Token{ - Term: []byte("Estudis"), - }, - }, - }, - } - - cache := registry.NewCache() - elisionFilter, err := cache.TokenFilterNamed(ElisionName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := elisionFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_filter_ca.go b/vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_filter_ca.go deleted file mode 100644 index d1d12e17..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_filter_ca.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ca - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_words_ca.go b/vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_words_ca.go deleted file mode 100644 index d88dfc31..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ca/stop_words_ca.go +++ /dev/null @@ -1,244 +0,0 @@ -package ca - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_ca" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var CatalanStopWords = []byte(`# Catalan stopwords from http://github.com/vcl/cue.language (Apache 2 Licensed) -a -abans -ací -ah -així -això -al -als -aleshores -algun -alguna -algunes -alguns -alhora -allà -allí -allò -altra -altre -altres -amb -ambdós -ambdues -apa -aquell -aquella -aquelles -aquells -aquest -aquesta -aquestes -aquests -aquí -baix -cada -cadascú -cadascuna -cadascunes -cadascuns -com -contra -d'un -d'una -d'unes -d'uns -dalt -de -del -dels -des -després -dins -dintre -donat -doncs -durant -e -eh -el -els -em -en -encara -ens -entre -érem -eren -éreu -es -és -esta -està -estàvem -estaven -estàveu -esteu -et -etc -ets -fins -fora -gairebé -ha -han -has -havia -he -hem -heu -hi -ho -i -igual -iguals -ja -l'hi -la -les -li -li'n -llavors -m'he -ma -mal -malgrat -mateix -mateixa -mateixes -mateixos -me -mentre -més -meu -meus -meva -meves -molt -molta -moltes -molts -mon -mons -n'he -n'hi -ne -ni -no -nogensmenys -només -nosaltres -nostra -nostre -nostres -o -oh -oi -on -pas -pel -pels -per -però -perquè -poc -poca -pocs -poques -potser -propi -qual -quals -quan -quant -que -què -quelcom -qui -quin -quina -quines -quins -s'ha -s'han -sa -semblant -semblants -ses -seu -seus -seva -seva -seves -si -sobre -sobretot -sóc -solament -sols -son -són -sons -sota -sou -t'ha -t'han -t'he -ta -tal -també -tampoc -tan -tant -tanta -tantes -teu -teus -teva -teves -ton -tons -tot -tota -totes -tots -un -una -unes -uns -us -va -vaig -vam -van -vas -veu -vosaltres -vostra -vostre -vostres -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(CatalanStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk.go b/vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk.go deleted file mode 100644 index f1841a6f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package cjk - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize" - "github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer" -) - -const AnalyzerName = "cjk" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - whitespaceTokenizer, err := cache.TokenizerNamed(whitespace_tokenizer.Name) - if err != nil { - return nil, err - } - normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKD) - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - bigramFilter, err := cache.TokenFilterNamed(BigramName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: whitespaceTokenizer, - TokenFilters: []analysis.TokenFilter{ - normalizeFilter, - toLowerFilter, - bigramFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk_test.go deleted file mode 100644 index 2c676928..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/analyzer_cjk_test.go +++ /dev/null @@ -1,620 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package cjk - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestCJKAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - input: []byte("こんにちは世界"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こん"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("んに"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - &analysis.Token{ - Term: []byte("にち"), - Type: analysis.Double, - Position: 3, - Start: 6, - End: 12, - }, - &analysis.Token{ - Term: []byte("ちは"), - Type: analysis.Double, - Position: 4, - Start: 9, - End: 15, - }, - &analysis.Token{ - Term: []byte("は世"), - Type: analysis.Double, - Position: 5, - Start: 12, - End: 18, - }, - &analysis.Token{ - Term: []byte("世界"), - Type: analysis.Double, - Position: 6, - Start: 15, - End: 21, - }, - }, - }, - { - input: []byte("一二三四五六七八九十"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("一二"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("二三"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - &analysis.Token{ - Term: []byte("三四"), - Type: analysis.Double, - Position: 3, - Start: 6, - End: 12, - }, - &analysis.Token{ - Term: []byte("四五"), - Type: analysis.Double, - Position: 4, - Start: 9, - End: 15, - }, - &analysis.Token{ - Term: []byte("五六"), - Type: analysis.Double, - Position: 5, - Start: 12, - End: 18, - }, - &analysis.Token{ - Term: []byte("六七"), - Type: analysis.Double, - Position: 6, - Start: 15, - End: 21, - }, - &analysis.Token{ - Term: []byte("七八"), - Type: analysis.Double, - Position: 7, - Start: 18, - End: 24, - }, - &analysis.Token{ - Term: []byte("八九"), - Type: analysis.Double, - Position: 8, - Start: 21, - End: 27, - }, - &analysis.Token{ - Term: []byte("九十"), - Type: analysis.Double, - Position: 9, - Start: 24, - End: 30, - }, - }, - }, - { - input: []byte("一 二三四 五六七八九 十"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("一"), - Type: analysis.Single, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("二三"), - Type: analysis.Double, - Position: 2, - Start: 4, - End: 10, - }, - &analysis.Token{ - Term: []byte("三四"), - Type: analysis.Double, - Position: 3, - Start: 7, - End: 13, - }, - &analysis.Token{ - Term: []byte("五六"), - Type: analysis.Double, - Position: 5, - Start: 14, - End: 20, - }, - &analysis.Token{ - Term: []byte("六七"), - Type: analysis.Double, - Position: 6, - Start: 17, - End: 23, - }, - &analysis.Token{ - Term: []byte("七八"), - Type: analysis.Double, - Position: 7, - Start: 20, - End: 26, - }, - &analysis.Token{ - Term: []byte("八九"), - Type: analysis.Double, - Position: 8, - Start: 23, - End: 29, - }, - &analysis.Token{ - Term: []byte("十"), - Type: analysis.Single, - Position: 10, - Start: 30, - End: 33, - }, - }, - }, - { - input: []byte("abc defgh ijklmn opqrstu vwxy z"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abc"), - Type: analysis.AlphaNumeric, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("defgh"), - Type: analysis.AlphaNumeric, - Position: 2, - Start: 4, - End: 9, - }, - &analysis.Token{ - Term: []byte("ijklmn"), - Type: analysis.AlphaNumeric, - Position: 3, - Start: 10, - End: 16, - }, - &analysis.Token{ - Term: []byte("opqrstu"), - Type: analysis.AlphaNumeric, - Position: 4, - Start: 17, - End: 24, - }, - &analysis.Token{ - Term: []byte("vwxy"), - Type: analysis.AlphaNumeric, - Position: 5, - Start: 25, - End: 29, - }, - &analysis.Token{ - Term: []byte("z"), - Type: analysis.AlphaNumeric, - Position: 6, - Start: 30, - End: 31, - }, - }, - }, - { - input: []byte("あい"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("あい"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - }, - }, - { - input: []byte("あい "), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("あい"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - }, - }, - { - input: []byte("test"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("test"), - Type: analysis.AlphaNumeric, - Position: 1, - Start: 0, - End: 4, - }, - }, - }, - { - input: []byte("test "), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("test"), - Type: analysis.AlphaNumeric, - Position: 1, - Start: 0, - End: 4, - }, - }, - }, - { - input: []byte("あいtest"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("あい"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("test"), - Type: analysis.AlphaNumeric, - Position: 3, - Start: 6, - End: 10, - }, - }, - }, - { - input: []byte("testあい "), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("test"), - Type: analysis.AlphaNumeric, - Position: 1, - Start: 0, - End: 4, - }, - &analysis.Token{ - Term: []byte("あい"), - Type: analysis.Double, - Position: 2, - Start: 4, - End: 10, - }, - }, - }, - { - input: []byte("あいうえおabcかきくけこ"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("あい"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("いう"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - &analysis.Token{ - Term: []byte("うえ"), - Type: analysis.Double, - Position: 3, - Start: 6, - End: 12, - }, - &analysis.Token{ - Term: []byte("えお"), - Type: analysis.Double, - Position: 4, - Start: 9, - End: 15, - }, - &analysis.Token{ - Term: []byte("abc"), - Type: analysis.AlphaNumeric, - Position: 6, - Start: 15, - End: 18, - }, - &analysis.Token{ - Term: []byte("かき"), - Type: analysis.Double, - Position: 7, - Start: 18, - End: 24, - }, - &analysis.Token{ - Term: []byte("きく"), - Type: analysis.Double, - Position: 8, - Start: 21, - End: 27, - }, - &analysis.Token{ - Term: []byte("くけ"), - Type: analysis.Double, - Position: 9, - Start: 24, - End: 30, - }, - &analysis.Token{ - Term: []byte("けこ"), - Type: analysis.Double, - Position: 10, - Start: 27, - End: 33, - }, - }, - }, - { - input: []byte("あいうえおabんcかきくけ こ"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("あい"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("いう"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - &analysis.Token{ - Term: []byte("うえ"), - Type: analysis.Double, - Position: 3, - Start: 6, - End: 12, - }, - &analysis.Token{ - Term: []byte("えお"), - Type: analysis.Double, - Position: 4, - Start: 9, - End: 15, - }, - &analysis.Token{ - Term: []byte("ab"), - Type: analysis.AlphaNumeric, - Position: 6, - Start: 15, - End: 17, - }, - &analysis.Token{ - Term: []byte("ん"), - Type: analysis.Single, - Position: 7, - Start: 17, - End: 20, - }, - &analysis.Token{ - Term: []byte("c"), - Type: analysis.AlphaNumeric, - Position: 8, - Start: 20, - End: 21, - }, - &analysis.Token{ - Term: []byte("かき"), - Type: analysis.Double, - Position: 9, - Start: 21, - End: 27, - }, - &analysis.Token{ - Term: []byte("きく"), - Type: analysis.Double, - Position: 10, - Start: 24, - End: 30, - }, - &analysis.Token{ - Term: []byte("くけ"), - Type: analysis.Double, - Position: 11, - Start: 27, - End: 33, - }, - &analysis.Token{ - Term: []byte("こ"), - Type: analysis.Single, - Position: 13, - Start: 34, - End: 37, - }, - }, - }, - { - input: []byte("一 روبرت موير"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("一"), - Type: analysis.Single, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("روبرت"), - Type: analysis.AlphaNumeric, - Position: 2, - Start: 4, - End: 14, - }, - &analysis.Token{ - Term: []byte("موير"), - Type: analysis.AlphaNumeric, - Position: 3, - Start: 15, - End: 23, - }, - }, - }, - { - input: []byte("一 رُوبرت موير"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("一"), - Type: analysis.Single, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("رُوبرت"), - Type: analysis.AlphaNumeric, - Position: 2, - Start: 4, - End: 16, - }, - &analysis.Token{ - Term: []byte("موير"), - Type: analysis.AlphaNumeric, - Position: 3, - Start: 17, - End: 25, - }, - }, - }, - { - input: []byte("𩬅艱鍟䇹愯瀛"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("𩬅艱"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 7, - }, - &analysis.Token{ - Term: []byte("艱鍟"), - Type: analysis.Double, - Position: 2, - Start: 4, - End: 10, - }, - &analysis.Token{ - Term: []byte("鍟䇹"), - Type: analysis.Double, - Position: 3, - Start: 7, - End: 13, - }, - &analysis.Token{ - Term: []byte("䇹愯"), - Type: analysis.Double, - Position: 4, - Start: 10, - End: 16, - }, - &analysis.Token{ - Term: []byte("愯瀛"), - Type: analysis.Double, - Position: 5, - Start: 13, - End: 19, - }, - }, - }, - { - input: []byte("一"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("一"), - Type: analysis.Single, - Position: 1, - Start: 0, - End: 3, - }, - }, - }, - { - input: []byte("一丁丂"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("一丁"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("丁丂"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - }, - }, - } - - cache := registry.NewCache() - for _, test := range tests { - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram.go b/vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram.go deleted file mode 100644 index 0596447e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package cjk - -import ( - "container/ring" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const BigramName = "cjk_bigram" - -type CJKBigramFilter struct { - outputUnigram bool -} - -func NewCJKBigramFilter(outputUnigram bool) *CJKBigramFilter { - return &CJKBigramFilter{ - outputUnigram: outputUnigram, - } -} - -func (s *CJKBigramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - r := ring.New(2) - itemsInRing := 0 - - rv := make(analysis.TokenStream, 0, len(input)) - - for _, token := range input { - if token.Type == analysis.Ideographic { - if itemsInRing > 0 { - // if items already buffered - // check to see if this is aligned - curr := r.Value.(*analysis.Token) - if token.Start-curr.End != 0 { - // not aligned flush - flushToken := s.flush(r, &itemsInRing) - if flushToken != nil { - rv = append(rv, flushToken) - } - } - } - // now we can add this token to the buffer - r = r.Next() - r.Value = token - if itemsInRing < 2 { - itemsInRing++ - } - if itemsInRing > 1 && s.outputUnigram { - unigram := s.buildUnigram(r, &itemsInRing) - if unigram != nil { - rv = append(rv, unigram) - } - } - bigramToken := s.outputBigram(r, &itemsInRing) - if bigramToken != nil { - rv = append(rv, bigramToken) - } - } else { - // flush anything already buffered - flushToken := s.flush(r, &itemsInRing) - if flushToken != nil { - rv = append(rv, flushToken) - } - // output this token as is - rv = append(rv, token) - } - } - - // deal with possible trailing unigram - if itemsInRing == 1 || s.outputUnigram { - if itemsInRing == 2 { - r = r.Next() - } - unigram := s.buildUnigram(r, &itemsInRing) - if unigram != nil { - rv = append(rv, unigram) - } - } - return rv -} - -func (s *CJKBigramFilter) flush(r *ring.Ring, itemsInRing *int) *analysis.Token { - var rv *analysis.Token - if *itemsInRing == 1 { - rv = s.buildUnigram(r, itemsInRing) - } - r.Value = nil - *itemsInRing = 0 - return rv -} - -func (s *CJKBigramFilter) outputBigram(r *ring.Ring, itemsInRing *int) *analysis.Token { - if *itemsInRing == 2 { - thisShingleRing := r.Move(-1) - shingledBytes := make([]byte, 0) - - // do first token - prev := thisShingleRing.Value.(*analysis.Token) - shingledBytes = append(shingledBytes, prev.Term...) - - // do second token - thisShingleRing = thisShingleRing.Next() - curr := thisShingleRing.Value.(*analysis.Token) - shingledBytes = append(shingledBytes, curr.Term...) - - token := analysis.Token{ - Type: analysis.Double, - Term: shingledBytes, - Position: prev.Position, - Start: prev.Start, - End: curr.End, - } - return &token - } - return nil -} - -func (s *CJKBigramFilter) buildUnigram(r *ring.Ring, itemsInRing *int) *analysis.Token { - if *itemsInRing == 2 { - thisShingleRing := r.Move(-1) - // do first token - prev := thisShingleRing.Value.(*analysis.Token) - token := analysis.Token{ - Type: analysis.Single, - Term: prev.Term, - Position: prev.Position, - Start: prev.Start, - End: prev.End, - } - return &token - } else if *itemsInRing == 1 { - // do first token - prev := r.Value.(*analysis.Token) - token := analysis.Token{ - Type: analysis.Single, - Term: prev.Term, - Position: prev.Position, - Start: prev.Start, - End: prev.End, - } - return &token - } - return nil -} - -func CJKBigramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - outputUnigram := false - outVal, ok := config["output_unigram"].(bool) - if ok { - outputUnigram = outVal - } - return NewCJKBigramFilter(outputUnigram), nil -} - -func init() { - registry.RegisterTokenFilter(BigramName, CJKBigramFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram_test.go deleted file mode 100644 index e30cacfe..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/cjk/cjk_bigram_test.go +++ /dev/null @@ -1,420 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package cjk - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestCJKBigramFilter(t *testing.T) { - - tests := []struct { - outputUnigram bool - input analysis.TokenStream - output analysis.TokenStream - }{ - { - outputUnigram: false, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こ"), - Type: analysis.Ideographic, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("ん"), - Type: analysis.Ideographic, - Position: 2, - Start: 5, - End: 7, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こ"), - Type: analysis.Single, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("ん"), - Type: analysis.Single, - Position: 2, - Start: 5, - End: 7, - }, - }, - }, - { - outputUnigram: false, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こ"), - Type: analysis.Ideographic, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("ん"), - Type: analysis.Ideographic, - Position: 2, - Start: 3, - End: 6, - }, - &analysis.Token{ - Term: []byte("に"), - Type: analysis.Ideographic, - Position: 3, - Start: 6, - End: 9, - }, - &analysis.Token{ - Term: []byte("ち"), - Type: analysis.Ideographic, - Position: 4, - Start: 9, - End: 12, - }, - &analysis.Token{ - Term: []byte("は"), - Type: analysis.Ideographic, - Position: 5, - Start: 12, - End: 15, - }, - &analysis.Token{ - Term: []byte("世"), - Type: analysis.Ideographic, - Position: 6, - Start: 15, - End: 18, - }, - &analysis.Token{ - Term: []byte("界"), - Type: analysis.Ideographic, - Position: 7, - Start: 18, - End: 21, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こん"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("んに"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - &analysis.Token{ - Term: []byte("にち"), - Type: analysis.Double, - Position: 3, - Start: 6, - End: 12, - }, - &analysis.Token{ - Term: []byte("ちは"), - Type: analysis.Double, - Position: 4, - Start: 9, - End: 15, - }, - &analysis.Token{ - Term: []byte("は世"), - Type: analysis.Double, - Position: 5, - Start: 12, - End: 18, - }, - &analysis.Token{ - Term: []byte("世界"), - Type: analysis.Double, - Position: 6, - Start: 15, - End: 21, - }, - }, - }, - { - outputUnigram: true, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こ"), - Type: analysis.Ideographic, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("ん"), - Type: analysis.Ideographic, - Position: 2, - Start: 3, - End: 6, - }, - &analysis.Token{ - Term: []byte("に"), - Type: analysis.Ideographic, - Position: 3, - Start: 6, - End: 9, - }, - &analysis.Token{ - Term: []byte("ち"), - Type: analysis.Ideographic, - Position: 4, - Start: 9, - End: 12, - }, - &analysis.Token{ - Term: []byte("は"), - Type: analysis.Ideographic, - Position: 5, - Start: 12, - End: 15, - }, - &analysis.Token{ - Term: []byte("世"), - Type: analysis.Ideographic, - Position: 6, - Start: 15, - End: 18, - }, - &analysis.Token{ - Term: []byte("界"), - Type: analysis.Ideographic, - Position: 7, - Start: 18, - End: 21, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こ"), - Type: analysis.Single, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("こん"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("ん"), - Type: analysis.Single, - Position: 2, - Start: 3, - End: 6, - }, - &analysis.Token{ - Term: []byte("んに"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - &analysis.Token{ - Term: []byte("に"), - Type: analysis.Single, - Position: 3, - Start: 6, - End: 9, - }, - &analysis.Token{ - Term: []byte("にち"), - Type: analysis.Double, - Position: 3, - Start: 6, - End: 12, - }, - &analysis.Token{ - Term: []byte("ち"), - Type: analysis.Single, - Position: 4, - Start: 9, - End: 12, - }, - &analysis.Token{ - Term: []byte("ちは"), - Type: analysis.Double, - Position: 4, - Start: 9, - End: 15, - }, - &analysis.Token{ - Term: []byte("は"), - Type: analysis.Single, - Position: 5, - Start: 12, - End: 15, - }, - &analysis.Token{ - Term: []byte("は世"), - Type: analysis.Double, - Position: 5, - Start: 12, - End: 18, - }, - &analysis.Token{ - Term: []byte("世"), - Type: analysis.Single, - Position: 6, - Start: 15, - End: 18, - }, - &analysis.Token{ - Term: []byte("世界"), - Type: analysis.Double, - Position: 6, - Start: 15, - End: 21, - }, - &analysis.Token{ - Term: []byte("界"), - Type: analysis.Single, - Position: 7, - Start: 18, - End: 21, - }, - }, - }, - { - outputUnigram: false, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こ"), - Type: analysis.Ideographic, - Position: 1, - Start: 0, - End: 3, - }, - &analysis.Token{ - Term: []byte("ん"), - Type: analysis.Ideographic, - Position: 2, - Start: 3, - End: 6, - }, - &analysis.Token{ - Term: []byte("に"), - Type: analysis.Ideographic, - Position: 3, - Start: 6, - End: 9, - }, - &analysis.Token{ - Term: []byte("ち"), - Type: analysis.Ideographic, - Position: 4, - Start: 9, - End: 12, - }, - &analysis.Token{ - Term: []byte("は"), - Type: analysis.Ideographic, - Position: 5, - Start: 12, - End: 15, - }, - &analysis.Token{ - Term: []byte("cat"), - Type: analysis.AlphaNumeric, - Position: 6, - Start: 12, - End: 15, - }, - &analysis.Token{ - Term: []byte("世"), - Type: analysis.Ideographic, - Position: 7, - Start: 18, - End: 21, - }, - &analysis.Token{ - Term: []byte("界"), - Type: analysis.Ideographic, - Position: 8, - Start: 21, - End: 24, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こん"), - Type: analysis.Double, - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("んに"), - Type: analysis.Double, - Position: 2, - Start: 3, - End: 9, - }, - &analysis.Token{ - Term: []byte("にち"), - Type: analysis.Double, - Position: 3, - Start: 6, - End: 12, - }, - &analysis.Token{ - Term: []byte("ちは"), - Type: analysis.Double, - Position: 4, - Start: 9, - End: 15, - }, - &analysis.Token{ - Term: []byte("cat"), - Type: analysis.AlphaNumeric, - Position: 6, - Start: 12, - End: 15, - }, - &analysis.Token{ - Term: []byte("世界"), - Type: analysis.Double, - Position: 7, - Start: 18, - End: 24, - }, - }, - }, - } - - for _, test := range tests { - cjkBigramFilter := NewCJKBigramFilter(test.outputUnigram) - actual := cjkBigramFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb.go deleted file mode 100644 index c33703a4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package ckb - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" - "github.com/blevesearch/bleve/registry" -) - -const AnalyzerName = "ckb" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - normCkbFilter, err := cache.TokenFilterNamed(NormalizeName) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopCkbFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerCkbFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - normCkbFilter, - toLowerFilter, - stopCkbFilter, - stemmerCkbFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb_test.go deleted file mode 100644 index 4c73801c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/analyzer_ckb_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package ckb - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestSoraniAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stop word removal - { - input: []byte("ئەم پیاوە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 2, - Start: 7, - End: 17, - }, - }, - }, - { - input: []byte("پیاوە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 1, - Start: 0, - End: 10, - }, - }, - }, - { - input: []byte("پیاو"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 1, - Start: 0, - End: 8, - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize.go deleted file mode 100644 index b264bb40..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ckb - -import ( - "bytes" - "unicode" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const NormalizeName = "normalize_ckb" - -const ( - Yeh = '\u064A' - DotlessYeh = '\u0649' - FarsiYeh = '\u06CC' - - Kaf = '\u0643' - Keheh = '\u06A9' - - Heh = '\u0647' - Ae = '\u06D5' - Zwnj = '\u200C' - HehDoachashmee = '\u06BE' - TehMarbuta = '\u0629' - - Reh = '\u0631' - Rreh = '\u0695' - RrehAbove = '\u0692' - - Tatweel = '\u0640' - Fathatan = '\u064B' - Dammatan = '\u064C' - Kasratan = '\u064D' - Fatha = '\u064E' - Damma = '\u064F' - Kasra = '\u0650' - Shadda = '\u0651' - Sukun = '\u0652' -) - -type SoraniNormalizeFilter struct { -} - -func NewSoraniNormalizeFilter() *SoraniNormalizeFilter { - return &SoraniNormalizeFilter{} -} - -func (s *SoraniNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - term := normalize(token.Term) - token.Term = term - } - return input -} - -func normalize(input []byte) []byte { - runes := bytes.Runes(input) - for i := 0; i < len(runes); i++ { - switch runes[i] { - case Yeh, DotlessYeh: - runes[i] = FarsiYeh - case Kaf: - runes[i] = Keheh - case Zwnj: - if i > 0 && runes[i-1] == Heh { - runes[i-1] = Ae - } - runes = analysis.DeleteRune(runes, i) - i-- - case Heh: - if i == len(runes)-1 { - runes[i] = Ae - } - case TehMarbuta: - runes[i] = Ae - case HehDoachashmee: - runes[i] = Heh - case Reh: - if i == 0 { - runes[i] = Rreh - } - case RrehAbove: - runes[i] = Rreh - case Tatweel, Kasratan, Dammatan, Fathatan, Fatha, Damma, Kasra, Shadda, Sukun: - runes = analysis.DeleteRune(runes, i) - i-- - default: - if unicode.In(runes[i], unicode.Cf) { - runes = analysis.DeleteRune(runes, i) - i-- - } - } - } - return analysis.BuildTermFromRunes(runes) -} - -func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewSoraniNormalizeFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize_test.go deleted file mode 100644 index 0d60d00e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_normalize_test.go +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ckb - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestSoraniNormalizeFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // test Y - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u064A"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06CC"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0649"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06CC"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06CC"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06CC"), - }, - }, - }, - // test K - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0643"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06A9"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06A9"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06A9"), - }, - }, - }, - // test H - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0647\u200C"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06D5"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0647\u200C\u06A9"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06D5\u06A9"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06BE"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0647"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0629"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u06D5"), - }, - }, - }, - // test final H - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0647\u0647\u0647"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0647\u0647\u06D5"), - }, - }, - }, - // test RR - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0692"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0695"), - }, - }, - }, - // test initial RR - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0631\u0631\u0631"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0695\u0631\u0631"), - }, - }, - }, - // test remove - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0640"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u064B"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u064C"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u064D"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u064E"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u064F"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0650"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0651"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0652"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u200C"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - // empty - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - soraniNormalizeFilter := NewSoraniNormalizeFilter() - for _, test := range tests { - actual := soraniNormalizeFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter.go deleted file mode 100644 index 68367a55..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ckb - -import ( - "bytes" - "unicode/utf8" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_ckb" - -type SoraniStemmerFilter struct { -} - -func NewSoraniStemmerFilter() *SoraniStemmerFilter { - return &SoraniStemmerFilter{} -} - -func (s *SoraniStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - // if not protected keyword, stem it - if !token.KeyWord { - stemmed := stem(token.Term) - token.Term = stemmed - } - } - return input -} - -func stem(input []byte) []byte { - inputLen := utf8.RuneCount(input) - - // postposition - if inputLen > 5 && bytes.HasSuffix(input, []byte("دا")) { - input = truncateRunes(input, 2) - inputLen = utf8.RuneCount(input) - } else if inputLen > 4 && bytes.HasSuffix(input, []byte("نا")) { - input = truncateRunes(input, 1) - inputLen = utf8.RuneCount(input) - } else if inputLen > 6 && bytes.HasSuffix(input, []byte("ەوە")) { - input = truncateRunes(input, 3) - inputLen = utf8.RuneCount(input) - } - - // possessive pronoun - if inputLen > 6 && - (bytes.HasSuffix(input, []byte("مان")) || - bytes.HasSuffix(input, []byte("یان")) || - bytes.HasSuffix(input, []byte("تان"))) { - input = truncateRunes(input, 3) - inputLen = utf8.RuneCount(input) - } - - // indefinite singular ezafe - if inputLen > 6 && bytes.HasSuffix(input, []byte("ێکی")) { - return truncateRunes(input, 3) - } else if inputLen > 7 && bytes.HasSuffix(input, []byte("یەکی")) { - return truncateRunes(input, 4) - } - - if inputLen > 5 && bytes.HasSuffix(input, []byte("ێک")) { - // indefinite singular - return truncateRunes(input, 2) - } else if inputLen > 6 && bytes.HasSuffix(input, []byte("یەک")) { - // indefinite singular - return truncateRunes(input, 3) - } else if inputLen > 6 && bytes.HasSuffix(input, []byte("ەکە")) { - // definite singular - return truncateRunes(input, 3) - } else if inputLen > 5 && bytes.HasSuffix(input, []byte("کە")) { - // definite singular - return truncateRunes(input, 2) - } else if inputLen > 7 && bytes.HasSuffix(input, []byte("ەکان")) { - // definite plural - return truncateRunes(input, 4) - } else if inputLen > 6 && bytes.HasSuffix(input, []byte("کان")) { - // definite plural - return truncateRunes(input, 3) - } else if inputLen > 7 && bytes.HasSuffix(input, []byte("یانی")) { - // indefinite plural ezafe - return truncateRunes(input, 4) - } else if inputLen > 6 && bytes.HasSuffix(input, []byte("انی")) { - // indefinite plural ezafe - return truncateRunes(input, 3) - } else if inputLen > 6 && bytes.HasSuffix(input, []byte("یان")) { - // indefinite plural - return truncateRunes(input, 3) - } else if inputLen > 5 && bytes.HasSuffix(input, []byte("ان")) { - // indefinite plural - return truncateRunes(input, 2) - } else if inputLen > 7 && bytes.HasSuffix(input, []byte("یانە")) { - // demonstrative plural - return truncateRunes(input, 4) - } else if inputLen > 6 && bytes.HasSuffix(input, []byte("انە")) { - // demonstrative plural - return truncateRunes(input, 3) - } else if inputLen > 5 && (bytes.HasSuffix(input, []byte("ایە")) || bytes.HasSuffix(input, []byte("ەیە"))) { - // demonstrative singular - return truncateRunes(input, 2) - } else if inputLen > 4 && bytes.HasSuffix(input, []byte("ە")) { - // demonstrative singular - return truncateRunes(input, 1) - } else if inputLen > 4 && bytes.HasSuffix(input, []byte("ی")) { - // absolute singular ezafe - return truncateRunes(input, 1) - } - return input -} - -func truncateRunes(input []byte, num int) []byte { - runes := bytes.Runes(input) - runes = runes[:len(runes)-num] - out := buildTermFromRunes(runes) - return out -} - -func buildTermFromRunes(runes []rune) []byte { - rv := make([]byte, 0, len(runes)*4) - for _, r := range runes { - runeBytes := make([]byte, utf8.RuneLen(r)) - utf8.EncodeRune(runeBytes, r) - rv = append(rv, runeBytes...) - } - return rv -} - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewSoraniStemmerFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter_test.go deleted file mode 100644 index 8567e67c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/sorani_stemmer_filter_test.go +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ckb - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizers/single_token" -) - -func TestSoraniStemmerFilter(t *testing.T) { - - // in order to match the lucene tests - // we will test with an analyzer, not just the stemmer - analyzer := analysis.Analyzer{ - Tokenizer: single_token.NewSingleTokenTokenizer(), - TokenFilters: []analysis.TokenFilter{ - NewSoraniNormalizeFilter(), - NewSoraniStemmerFilter(), - }, - } - - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { // -ek - input: []byte("پیاوێک"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - { // -yek - input: []byte("دەرگایەک"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("دەرگا"), - Position: 1, - Start: 0, - End: 16, - }, - }, - }, - { // -aka - input: []byte("پیاوەكە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // -ka - input: []byte("دەرگاكە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("دەرگا"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // -a - input: []byte("کتاویە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("کتاوی"), - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - { // -ya - input: []byte("دەرگایە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("دەرگا"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // -An - input: []byte("پیاوان"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - { // -yAn - input: []byte("دەرگایان"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("دەرگا"), - Position: 1, - Start: 0, - End: 16, - }, - }, - }, - { // -akAn - input: []byte("پیاوەکان"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 1, - Start: 0, - End: 16, - }, - }, - }, - { // -kAn - input: []byte("دەرگاکان"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("دەرگا"), - Position: 1, - Start: 0, - End: 16, - }, - }, - }, - { // -Ana - input: []byte("پیاوانە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پیاو"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // -yAna - input: []byte("دەرگایانە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("دەرگا"), - Position: 1, - Start: 0, - End: 18, - }, - }, - }, - { // Ezafe singular - input: []byte("هۆتیلی"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هۆتیل"), - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - { // Ezafe indefinite - input: []byte("هۆتیلێکی"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هۆتیل"), - Position: 1, - Start: 0, - End: 16, - }, - }, - }, - { // Ezafe plural - input: []byte("هۆتیلانی"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هۆتیل"), - Position: 1, - Start: 0, - End: 16, - }, - }, - }, - { // -awa - input: []byte("دوورەوە"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("دوور"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // -dA - input: []byte("نیوەشەودا"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("نیوەشەو"), - Position: 1, - Start: 0, - End: 18, - }, - }, - }, - { // -A - input: []byte("سۆرانا"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("سۆران"), - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - { // -mAn - input: []byte("پارەمان"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پارە"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // -tAn - input: []byte("پارەتان"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پارە"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // -yAn - input: []byte("پارەیان"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("پارە"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { // empty - input: []byte(""), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - Position: 1, - Start: 0, - End: 0, - }, - }, - }, - } - - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("for input %s(% x)", test.input, test.input) - t.Errorf("\texpected:") - for _, token := range test.output { - t.Errorf("\t\t%v %s(% x)", token, token.Term, token.Term) - } - t.Errorf("\tactual:") - for _, token := range actual { - t.Errorf("\t\t%v %s(% x)", token, token.Term, token.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_filter_ckb.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_filter_ckb.go deleted file mode 100644 index bf8c5170..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_filter_ckb.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ckb - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_words_ckb.go b/vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_words_ckb.go deleted file mode 100644 index 707a3b16..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ckb/stop_words_ckb.go +++ /dev/null @@ -1,160 +0,0 @@ -package ckb - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_ckb" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var SoraniStopWords = []byte(`# set of kurdish stopwords -# note these have been normalized with our scheme (e represented with U+06D5, etc) -# constructed from: -# * Fig 5 of "Building A Test Collection For Sorani Kurdish" (Esmaili et al) -# * "Sorani Kurdish: A Reference Grammar with selected readings" (Thackston) -# * Corpus-based analysis of 77M word Sorani collection: wikipedia, news, blogs, etc - -# and -و -# which -کە -# of -ی -# made/did -کرد -# that/which -ئەوەی -# on/head -سەر -# two -دوو -# also -هەروەها -# from/that -لەو -# makes/does -دەکات -# some -چەند -# every -هەر - -# demonstratives -# that -ئەو -# this -ئەم - -# personal pronouns -# I -من -# we -ئێمە -# you -تۆ -# you -ئێوە -# he/she/it -ئەو -# they -ئەوان - -# prepositions -# to/with/by -بە -پێ -# without -بەبێ -# along with/while/during -بەدەم -# in the opinion of -بەلای -# according to -بەپێی -# before -بەرلە -# in the direction of -بەرەوی -# in front of/toward -بەرەوە -# before/in the face of -بەردەم -# without -بێ -# except for -بێجگە -# for -بۆ -# on/in -دە -تێ -# with -دەگەڵ -# after -دوای -# except for/aside from -جگە -# in/from -لە -لێ -# in front of/before/because of -لەبەر -# between/among -لەبەینی -# concerning/about -لەبابەت -# concerning -لەبارەی -# instead of -لەباتی -# beside -لەبن -# instead of -لەبرێتی -# behind -لەدەم -# with/together with -لەگەڵ -# by -لەلایەن -# within -لەناو -# between/among -لەنێو -# for the sake of -لەپێناوی -# with respect to -لەرەوی -# by means of/for -لەرێ -# for the sake of -لەرێگا -# on/on top of/according to -لەسەر -# under -لەژێر -# between/among -ناو -# between/among -نێوان -# after -پاش -# before -پێش -# like -وەک -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(SoraniStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_filter_cs.go b/vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_filter_cs.go deleted file mode 100644 index 0ca858c5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_filter_cs.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package cs - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_words_cs.go b/vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_words_cs.go deleted file mode 100644 index 71f9c43e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/cs/stop_words_cs.go +++ /dev/null @@ -1,196 +0,0 @@ -package cs - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_cs" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var CzechStopWords = []byte(`a -s -k -o -i -u -v -z -dnes -cz -tímto -budeš -budem -byli -jseš -můj -svým -ta -tomto -tohle -tuto -tyto -jej -zda -proč -máte -tato -kam -tohoto -kdo -kteří -mi -nám -tom -tomuto -mít -nic -proto -kterou -byla -toho -protože -asi -ho -naši -napište -re -což -tím -takže -svých -její -svými -jste -aj -tu -tedy -teto -bylo -kde -ke -pravé -ji -nad -nejsou -či -pod -téma -mezi -přes -ty -pak -vám -ani -když -však -neg -jsem -tento -článku -články -aby -jsme -před -pta -jejich -byl -ještě -až -bez -také -pouze -první -vaše -která -nás -nový -tipy -pokud -může -strana -jeho -své -jiné -zprávy -nové -není -vás -jen -podle -zde -už -být -více -bude -již -než -který -by -které -co -nebo -ten -tak -má -při -od -po -jsou -jak -další -ale -si -se -ve -to -jako -za -zpět -ze -do -pro -je -na -atd -atp -jakmile -přičemž -já -on -ona -ono -oni -ony -my -vy -jí -ji -mě -mne -jemu -tomu -těm -těmu -němu -němuž -jehož -jíž -jelikož -jež -jakož -načež -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(CzechStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da.go b/vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da.go deleted file mode 100644 index a348e07c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package da - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" - "github.com/blevesearch/bleve/registry" -) - -const AnalyzerName = "da" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopDaFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerDaFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopDaFilter, - stemmerDaFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da_test.go deleted file mode 100644 index d37af0fc..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/da/analyzer_da_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package da - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestDanishAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("undersøg"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("undersøg"), - Position: 1, - Start: 0, - End: 9, - }, - }, - }, - { - input: []byte("undersøgelse"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("undersøg"), - Position: 1, - Start: 0, - End: 13, - }, - }, - }, - // stop word - { - input: []byte("på"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/da/stemmer_da.go b/vendor/github.com/blevesearch/bleve/analysis/language/da/stemmer_da.go deleted file mode 100644 index 56e6ba02..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/da/stemmer_da.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package da - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_da" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("da") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/da/stop_filter_da.go b/vendor/github.com/blevesearch/bleve/analysis/language/da/stop_filter_da.go deleted file mode 100644 index 7989c6ad..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/da/stop_filter_da.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package da - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/da/stop_words_da.go b/vendor/github.com/blevesearch/bleve/analysis/language/da/stop_words_da.go deleted file mode 100644 index 63a407a0..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/da/stop_words_da.go +++ /dev/null @@ -1,134 +0,0 @@ -package da - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_da" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var DanishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Danish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - -og | and -i | in -jeg | I -det | that (dem. pronoun)/it (pers. pronoun) -at | that (in front of a sentence)/to (with infinitive) -en | a/an -den | it (pers. pronoun)/that (dem. pronoun) -til | to/at/for/until/against/by/of/into, more -er | present tense of "to be" -som | who, as -på | on/upon/in/on/at/to/after/of/with/for, on -de | they -med | with/by/in, along -han | he -af | of/by/from/off/for/in/with/on, off -for | at/for/to/from/by/of/ago, in front/before, because -ikke | not -der | who/which, there/those -var | past tense of "to be" -mig | me/myself -sig | oneself/himself/herself/itself/themselves -men | but -et | a/an/one, one (number), someone/somebody/one -har | present tense of "to have" -om | round/about/for/in/a, about/around/down, if -vi | we -min | my -havde | past tense of "to have" -ham | him -hun | she -nu | now -over | over/above/across/by/beyond/past/on/about, over/past -da | then, when/as/since -fra | from/off/since, off, since -du | you -ud | out -sin | his/her/its/one's -dem | them -os | us/ourselves -op | up -man | you/one -hans | his -hvor | where -eller | or -hvad | what -skal | must/shall etc. -selv | myself/youself/herself/ourselves etc., even -her | here -alle | all/everyone/everybody etc. -vil | will (verb) -blev | past tense of "to stay/to remain/to get/to become" -kunne | could -ind | in -når | when -være | present tense of "to be" -dog | however/yet/after all -noget | something -ville | would -jo | you know/you see (adv), yes -deres | their/theirs -efter | after/behind/according to/for/by/from, later/afterwards -ned | down -skulle | should -denne | this -end | than -dette | this -mit | my/mine -også | also -under | under/beneath/below/during, below/underneath -have | have -dig | you -anden | other -hende | her -mine | my -alt | everything -meget | much/very, plenty of -sit | his, her, its, one's -sine | his, her, its, one's -vor | our -mod | against -disse | these -hvis | if -din | your/yours -nogle | some -hos | by/at -blive | be/become -mange | many -ad | by/through -bliver | present tense of "to be/to become" -hendes | her/hers -været | be -thi | for (conj) -jer | you -sådan | such, like this/like that -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(DanishStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de.go b/vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de.go deleted file mode 100644 index 1f0169ac..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package de - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" - "github.com/blevesearch/bleve/registry" -) - -const AnalyzerName = "de" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopDeFilter, err := cache.TokenFilterNamed(NormalizeName) - if err != nil { - return nil, err - } - normalizeDeFilter, err := cache.TokenFilterNamed(NormalizeName) - if err != nil { - return nil, err - } - stemmerDeFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopDeFilter, - normalizeDeFilter, - stemmerDeFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de_test.go deleted file mode 100644 index 0d4bf55c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/de/analyzer_de_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package de - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestGermanAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - input: []byte("Tisch"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("tisch"), - Position: 1, - Start: 0, - End: 5, - }, - }, - }, - { - input: []byte("Tische"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("tisch"), - Position: 1, - Start: 0, - End: 6, - }, - }, - }, - { - input: []byte("Tischen"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("tisch"), - Position: 1, - Start: 0, - End: 7, - }, - }, - }, - // german specials - { - input: []byte("Schaltflächen"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("schaltflach"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - { - input: []byte("Schaltflaechen"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("schaltflach"), - Position: 1, - Start: 0, - End: 14, - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize.go b/vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize.go deleted file mode 100644 index 47cc97b4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package de - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const NormalizeName = "normalize_de" - -const ( - N = 0 /* ordinary state */ - V = 1 /* stops 'u' from entering umlaut state */ - U = 2 /* umlaut state, allows e-deletion */ -) - -type GermanNormalizeFilter struct { -} - -func NewGermanNormalizeFilter() *GermanNormalizeFilter { - return &GermanNormalizeFilter{} -} - -func (s *GermanNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - term := normalize(token.Term) - token.Term = term - } - return input -} - -func normalize(input []byte) []byte { - state := N - runes := bytes.Runes(input) - for i := 0; i < len(runes); i++ { - switch runes[i] { - case 'a', 'o': - state = U - case 'u': - if state == N { - state = U - } else { - state = V - } - case 'e': - if state == U { - runes = analysis.DeleteRune(runes, i) - i-- - } - state = V - case 'i', 'q', 'y': - state = V - case 'ä': - runes[i] = 'a' - state = V - case 'ö': - runes[i] = 'o' - state = V - case 'ü': - runes[i] = 'u' - state = V - case 'ß': - runes[i] = 's' - i++ - // newrunes := make([]rune, len(runes)+1) - // copy(newrunes, runes) - // runes = newrunes - // runes[i] = 's' - runes = analysis.InsertRune(runes, i, 's') - state = N - default: - state = N - } - } - return analysis.BuildTermFromRunes(runes) -} - -func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewGermanNormalizeFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize_test.go deleted file mode 100644 index 8315a192..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/de/german_normalize_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package de - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestGermanNormalizeFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // Tests that a/o/u + e is equivalent to the umlaut form - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Schaltflächen"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Schaltflachen"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Schaltflaechen"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Schaltflachen"), - }, - }, - }, - // Tests the specific heuristic that ue is not folded after a vowel or q. - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("dauer"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("dauer"), - }, - }, - }, - // Tests german specific folding of sharp-s - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("weißbier"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("weissbier"), - }, - }, - }, - // empty - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - germanNormalizeFilter := NewGermanNormalizeFilter() - for _, test := range tests { - actual := germanNormalizeFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected %s(% x), got %s(% x)", test.output[0].Term, test.output[0].Term, actual[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/de/stemmer_de.go b/vendor/github.com/blevesearch/bleve/analysis/language/de/stemmer_de.go deleted file mode 100644 index a37c7159..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/de/stemmer_de.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package de - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_de" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("de") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/de/stop_filter_de.go b/vendor/github.com/blevesearch/bleve/analysis/language/de/stop_filter_de.go deleted file mode 100644 index 2824cd19..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/de/stop_filter_de.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package de - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/de/stop_words_de.go b/vendor/github.com/blevesearch/bleve/analysis/language/de/stop_words_de.go deleted file mode 100644 index b71c8f70..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/de/stop_words_de.go +++ /dev/null @@ -1,318 +0,0 @@ -package de - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_de" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var GermanStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A German stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | The number of forms in this list is reduced significantly by passing it - | through the German stemmer. - - -aber | but - -alle | all -allem -allen -aller -alles - -als | than, as -also | so -am | an + dem -an | at - -ander | other -andere -anderem -anderen -anderer -anderes -anderm -andern -anderr -anders - -auch | also -auf | on -aus | out of -bei | by -bin | am -bis | until -bist | art -da | there -damit | with it -dann | then - -der | the -den -des -dem -die -das - -daß | that - -derselbe | the same -derselben -denselben -desselben -demselben -dieselbe -dieselben -dasselbe - -dazu | to that - -dein | thy -deine -deinem -deinen -deiner -deines - -denn | because - -derer | of those -dessen | of him - -dich | thee -dir | to thee -du | thou - -dies | this -diese -diesem -diesen -dieser -dieses - - -doch | (several meanings) -dort | (over) there - - -durch | through - -ein | a -eine -einem -einen -einer -eines - -einig | some -einige -einigem -einigen -einiger -einiges - -einmal | once - -er | he -ihn | him -ihm | to him - -es | it -etwas | something - -euer | your -eure -eurem -euren -eurer -eures - -für | for -gegen | towards -gewesen | p.p. of sein -hab | have -habe | have -haben | have -hat | has -hatte | had -hatten | had -hier | here -hin | there -hinter | behind - -ich | I -mich | me -mir | to me - - -ihr | you, to her -ihre -ihrem -ihren -ihrer -ihres -euch | to you - -im | in + dem -in | in -indem | while -ins | in + das -ist | is - -jede | each, every -jedem -jeden -jeder -jedes - -jene | that -jenem -jenen -jener -jenes - -jetzt | now -kann | can - -kein | no -keine -keinem -keinen -keiner -keines - -können | can -könnte | could -machen | do -man | one - -manche | some, many a -manchem -manchen -mancher -manches - -mein | my -meine -meinem -meinen -meiner -meines - -mit | with -muss | must -musste | had to -nach | to(wards) -nicht | not -nichts | nothing -noch | still, yet -nun | now -nur | only -ob | whether -oder | or -ohne | without -sehr | very - -sein | his -seine -seinem -seinen -seiner -seines - -selbst | self -sich | herself - -sie | they, she -ihnen | to them - -sind | are -so | so - -solche | such -solchem -solchen -solcher -solches - -soll | shall -sollte | should -sondern | but -sonst | else -über | over -um | about, around -und | and - -uns | us -unse -unsem -unsen -unser -unses - -unter | under -viel | much -vom | von + dem -von | from -vor | before -während | while -war | was -waren | were -warst | wast -was | what -weg | away, off -weil | because -weiter | further - -welche | which -welchem -welchen -welcher -welches - -wenn | when -werde | will -werden | will -wie | how -wieder | again -will | want -wir | we -wird | will -wirst | willst -wo | where -wollen | want -wollte | wanted -würde | would -würden | would -zu | to -zum | zu + dem -zur | zu + der -zwar | indeed -zwischen | between - -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(GermanStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/el/stop_filter_el.go b/vendor/github.com/blevesearch/bleve/analysis/language/el/stop_filter_el.go deleted file mode 100644 index ca184541..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/el/stop_filter_el.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package el - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/el/stop_words_el.go b/vendor/github.com/blevesearch/bleve/analysis/language/el/stop_words_el.go deleted file mode 100644 index 10755d28..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/el/stop_words_el.go +++ /dev/null @@ -1,102 +0,0 @@ -package el - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_el" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var GreekStopWords = []byte(`# Lucene Greek Stopwords list -# Note: by default this file is used after GreekLowerCaseFilter, -# so when modifying this file use 'σ' instead of 'ς' -ο -η -το -οι -τα -του -τησ -των -τον -την -και -κι -κ -ειμαι -εισαι -ειναι -ειμαστε -ειστε -στο -στον -στη -στην -μα -αλλα -απο -για -προσ -με -σε -ωσ -παρα -αντι -κατα -μετα -θα -να -δε -δεν -μη -μην -επι -ενω -εαν -αν -τοτε -που -πωσ -ποιοσ -ποια -ποιο -ποιοι -ποιεσ -ποιων -ποιουσ -αυτοσ -αυτη -αυτο -αυτοι -αυτων -αυτουσ -αυτεσ -αυτα -εκεινοσ -εκεινη -εκεινο -εκεινοι -εκεινεσ -εκεινα -εκεινων -εκεινουσ -οπωσ -ομωσ -ισωσ -οσο -οτι -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(GreekStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en.go b/vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en.go deleted file mode 100644 index 18bd764f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package en - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/token_filters/porter" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" -) - -const AnalyzerName = "en" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - possEnFilter, err := cache.TokenFilterNamed(PossessiveName) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopEnFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerEnFilter, err := cache.TokenFilterNamed(porter.Name) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - possEnFilter, - toLowerFilter, - stopEnFilter, - stemmerEnFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en_test.go deleted file mode 100644 index 08e4e457..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/analyzer_en_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package en - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestEnglishAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("books"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("book"), - Position: 1, - Start: 0, - End: 5, - }, - }, - }, - { - input: []byte("book"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("book"), - Position: 1, - Start: 0, - End: 4, - }, - }, - }, - // stop word removal - { - input: []byte("the"), - output: analysis.TokenStream{}, - }, - // possessive removal - { - input: []byte("steven's"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("steven"), - Position: 1, - Start: 0, - End: 8, - }, - }, - }, - { - input: []byte("steven\u2019s"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("steven"), - Position: 1, - Start: 0, - End: 10, - }, - }, - }, - { - input: []byte("steven\uFF07s"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("steven"), - Position: 1, - Start: 0, - End: 10, - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en.go b/vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en.go deleted file mode 100644 index 84a62c2f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package en - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const PossessiveName = "possessive_en" - -const rightSingleQuotationMark = '’' -const apostrophe = '\'' -const fullWidthApostrophe = '＇' - -const apostropheChars = rightSingleQuotationMark + apostrophe + fullWidthApostrophe - -type PossessiveFilter struct { -} - -func NewPossessiveFilter() *PossessiveFilter { - return &PossessiveFilter{} -} - -func (s *PossessiveFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - runes := bytes.Runes(token.Term) - if len(runes) >= 2 { - secondToLastRune := runes[len(runes)-2] - lastRune := runes[len(runes)-1] - if (secondToLastRune == rightSingleQuotationMark || - secondToLastRune == apostrophe || - secondToLastRune == fullWidthApostrophe) && - (lastRune == 's' || lastRune == 'S') { - token.Term = analysis.TruncateRunes(token.Term, 2) - } - } - } - return input -} - -func PossessiveFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewPossessiveFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(PossessiveName, PossessiveFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en_test.go deleted file mode 100644 index e434bf5d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/possessive_filter_en_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package en - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestEnglishPossessiveFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("marty's"), - }, - &analysis.Token{ - Term: []byte("MARTY'S"), - }, - &analysis.Token{ - Term: []byte("marty’s"), - }, - &analysis.Token{ - Term: []byte("MARTY’S"), - }, - &analysis.Token{ - Term: []byte("marty＇s"), - }, - &analysis.Token{ - Term: []byte("MARTY＇S"), - }, - &analysis.Token{ - Term: []byte("m"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("marty"), - }, - &analysis.Token{ - Term: []byte("MARTY"), - }, - &analysis.Token{ - Term: []byte("marty"), - }, - &analysis.Token{ - Term: []byte("MARTY"), - }, - &analysis.Token{ - Term: []byte("marty"), - }, - &analysis.Token{ - Term: []byte("MARTY"), - }, - &analysis.Token{ - Term: []byte("m"), - }, - }, - }, - } - - cache := registry.NewCache() - stemmerFilter, err := cache.TokenFilterNamed(PossessiveName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := stemmerFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en.go b/vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en.go deleted file mode 100644 index 695b6381..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package en - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_en" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("en") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en_test.go deleted file mode 100644 index 6506968e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/stemmer_en_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package en - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestEnglishStemmer(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("walking"), - }, - &analysis.Token{ - Term: []byte("talked"), - }, - &analysis.Token{ - Term: []byte("business"), - }, - &analysis.Token{ - Term: []byte("protected"), - KeyWord: true, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("walk"), - }, - &analysis.Token{ - Term: []byte("talk"), - }, - &analysis.Token{ - Term: []byte("busi"), - }, - &analysis.Token{ - Term: []byte("protected"), - KeyWord: true, - }, - }, - }, - } - - cache := registry.NewCache() - stemmerFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := stemmerFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/en/stop_filter_en.go b/vendor/github.com/blevesearch/bleve/analysis/language/en/stop_filter_en.go deleted file mode 100644 index 1d3e249d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/en/stop_filter_en.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package en - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es.go b/vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es.go deleted file mode 100644 index bcc39891..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package es - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "es" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopEsFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerEsFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopEsFilter, - stemmerEsFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es_test.go deleted file mode 100644 index 67b31b2f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/es/analyzer_es_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package es - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestSpanishAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("chicana"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chican"), - Position: 1, - Start: 0, - End: 7, - }, - }, - }, - { - input: []byte("chicano"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chican"), - Position: 1, - Start: 0, - End: 7, - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/es/stemmer_es.go b/vendor/github.com/blevesearch/bleve/analysis/language/es/stemmer_es.go deleted file mode 100644 index 6b4c64ad..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/es/stemmer_es.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package es - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_es" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("es") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/es/stop_filter_es.go b/vendor/github.com/blevesearch/bleve/analysis/language/es/stop_filter_es.go deleted file mode 100644 index 0a950544..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/es/stop_filter_es.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package es - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/es/stop_words_es.go b/vendor/github.com/blevesearch/bleve/analysis/language/es/stop_words_es.go deleted file mode 100644 index b699daf4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/es/stop_words_es.go +++ /dev/null @@ -1,380 +0,0 @@ -package es - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_es" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var SpanishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Spanish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - - | The following is a ranked list (commonest to rarest) of stopwords - | deriving from a large sample of text. - - | Extra words have been added at the end. - -de | from, of -la | the, her -que | who, that -el | the -en | in -y | and -a | to -los | the, them -del | de + el -se | himself, from him etc -las | the, them -por | for, by, etc -un | a -para | for -con | with -no | no -una | a -su | his, her -al | a + el - | es from SER -lo | him -como | how -más | more -pero | pero -sus | su plural -le | to him, her -ya | already -o | or - | fue from SER -este | this - | ha from HABER -sí | himself etc -porque | because -esta | this - | son from SER -entre | between - | está from ESTAR -cuando | when -muy | very -sin | without -sobre | on - | ser from SER - | tiene from TENER -también | also -me | me -hasta | until -hay | there is/are -donde | where - | han from HABER -quien | whom, that - | están from ESTAR - | estado from ESTAR -desde | from -todo | all -nos | us -durante | during - | estados from ESTAR -todos | all -uno | a -les | to them -ni | nor -contra | against -otros | other - | fueron from SER -ese | that -eso | that - | había from HABER -ante | before -ellos | they -e | and (variant of y) -esto | this -mí | me -antes | before -algunos | some -qué | what? -unos | a -yo | I -otro | other -otras | other -otra | other -él | he -tanto | so much, many -esa | that -estos | these -mucho | much, many -quienes | who -nada | nothing -muchos | many -cual | who - | sea from SER -poco | few -ella | she -estar | to be - | haber from HABER -estas | these - | estaba from ESTAR - | estamos from ESTAR -algunas | some -algo | something -nosotros | we - - | other forms - -mi | me -mis | mi plural -tú | thou -te | thee -ti | thee -tu | thy -tus | tu plural -ellas | they -nosotras | we -vosotros | you -vosotras | you -os | you -mío | mine -mía | -míos | -mías | -tuyo | thine -tuya | -tuyos | -tuyas | -suyo | his, hers, theirs -suya | -suyos | -suyas | -nuestro | ours -nuestra | -nuestros | -nuestras | -vuestro | yours -vuestra | -vuestros | -vuestras | -esos | those -esas | those - - | forms of estar, to be (not including the infinitive): -estoy -estás -está -estamos -estáis -están -esté -estés -estemos -estéis -estén -estaré -estarás -estará -estaremos -estaréis -estarán -estaría -estarías -estaríamos -estaríais -estarían -estaba -estabas -estábamos -estabais -estaban -estuve -estuviste -estuvo -estuvimos -estuvisteis -estuvieron -estuviera -estuvieras -estuviéramos -estuvierais -estuvieran -estuviese -estuvieses -estuviésemos -estuvieseis -estuviesen -estando -estado -estada -estados -estadas -estad - - | forms of haber, to have (not including the infinitive): -he -has -ha -hemos -habéis -han -haya -hayas -hayamos -hayáis -hayan -habré -habrás -habrá -habremos -habréis -habrán -habría -habrías -habríamos -habríais -habrían -había -habías -habíamos -habíais -habían -hube -hubiste -hubo -hubimos -hubisteis -hubieron -hubiera -hubieras -hubiéramos -hubierais -hubieran -hubiese -hubieses -hubiésemos -hubieseis -hubiesen -habiendo -habido -habida -habidos -habidas - - | forms of ser, to be (not including the infinitive): -soy -eres -es -somos -sois -son -sea -seas -seamos -seáis -sean -seré -serás -será -seremos -seréis -serán -sería -serías -seríamos -seríais -serían -era -eras -éramos -erais -eran -fui -fuiste -fue -fuimos -fuisteis -fueron -fuera -fueras -fuéramos -fuerais -fueran -fuese -fueses -fuésemos -fueseis -fuesen -siendo -sido - | sed also means 'thirst' - - | forms of tener, to have (not including the infinitive): -tengo -tienes -tiene -tenemos -tenéis -tienen -tenga -tengas -tengamos -tengáis -tengan -tendré -tendrás -tendrá -tendremos -tendréis -tendrán -tendría -tendrías -tendríamos -tendríais -tendrían -tenía -tenías -teníamos -teníais -tenían -tuve -tuviste -tuvo -tuvimos -tuvisteis -tuvieron -tuviera -tuvieras -tuviéramos -tuvierais -tuvieran -tuviese -tuvieses -tuviésemos -tuvieseis -tuviesen -teniendo -tenido -tenida -tenidos -tenidas -tened - -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(SpanishStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_filter_eu.go b/vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_filter_eu.go deleted file mode 100644 index 18d28274..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_filter_eu.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package eu - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_words_eu.go b/vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_words_eu.go deleted file mode 100644 index dcb3f7b3..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/eu/stop_words_eu.go +++ /dev/null @@ -1,123 +0,0 @@ -package eu - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_eu" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var BasqueStopWords = []byte(`# example set of basque stopwords -al -anitz -arabera -asko -baina -bat -batean -batek -bati -batzuei -batzuek -batzuetan -batzuk -bera -beraiek -berau -berauek -bere -berori -beroriek -beste -bezala -da -dago -dira -ditu -du -dute -edo -egin -ere -eta -eurak -ez -gainera -gu -gutxi -guzti -haiei -haiek -haietan -hainbeste -hala -han -handik -hango -hara -hari -hark -hartan -hau -hauei -hauek -hauetan -hemen -hemendik -hemengo -hi -hona -honek -honela -honetan -honi -hor -hori -horiei -horiek -horietan -horko -horra -horrek -horrela -horretan -horri -hortik -hura -izan -ni -noiz -nola -non -nondik -nongo -nor -nora -ze -zein -zen -zenbait -zenbat -zer -zergatik -ziren -zituen -zu -zuek -zuen -zuten -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(BasqueStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa.go b/vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa.go deleted file mode 100644 index 7d9a525c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package fa - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/char_filters/zero_width_non_joiner" - "github.com/blevesearch/bleve/analysis/language/ar" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "fa" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - zFilter, err := cache.CharFilterNamed(zero_width_non_joiner.Name) - if err != nil { - return nil, err - } - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - normArFilter, err := cache.TokenFilterNamed(ar.NormalizeName) - if err != nil { - return nil, err - } - normFaFilter, err := cache.TokenFilterNamed(NormalizeName) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopFaFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - CharFilters: []analysis.CharFilter{ - zFilter, - }, - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - normArFilter, - normFaFilter, - stopFaFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa_test.go deleted file mode 100644 index 1f95a53d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fa/analyzer_fa_test.go +++ /dev/null @@ -1,681 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package fa - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestPersianAnalyzerVerbs(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // active present indicative - { - input: []byte("می‌خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active preterite indicative - { - input: []byte("خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active imperfective preterite indicative - { - input: []byte("می‌خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active future indicative - { - input: []byte("خواهد خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active present progressive indicative - { - input: []byte("دارد می‌خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active preterite progressive indicative - { - input: []byte("داشت می‌خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active perfect indicative - { - input: []byte("خورده‌است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective perfect indicative - { - input: []byte("می‌خورده‌است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active pluperfect indicative - { - input: []byte("خورده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective pluperfect indicative - { - input: []byte("می‌خورده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active preterite subjunctive - { - input: []byte("خورده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective preterite subjunctive - { - input: []byte("می‌خورده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active pluperfect subjunctive - { - input: []byte("خورده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective pluperfect subjunctive - { - input: []byte("می‌خورده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive present indicative - { - input: []byte("خورده می‌شود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive preterite indicative - { - input: []byte("خورده شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective preterite indicative - { - input: []byte("خورده می‌شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive perfect indicative - { - input: []byte("خورده شده‌است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective perfect indicative - { - input: []byte("خورده می‌شده‌است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive pluperfect indicative - { - input: []byte("خورده شده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective pluperfect indicative - { - input: []byte("خورده می‌شده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive future indicative - { - input: []byte("خورده خواهد شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive present progressive indicative - { - input: []byte("دارد خورده می‌شود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive preterite progressive indicative - { - input: []byte("داشت خورده می‌شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive present subjunctive - { - input: []byte("خورده شود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive preterite subjunctive - { - input: []byte("خورده شده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective preterite subjunctive - { - input: []byte("خورده می‌شده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive pluperfect subjunctive - { - input: []byte("خورده شده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective pluperfect subjunctive - { - input: []byte("خورده می‌شده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active present subjunctive - { - input: []byte("بخورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("بخورد"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} - -func TestPersianAnalyzerVerbsDefective(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // active present indicative - { - input: []byte("مي خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active preterite indicative - { - input: []byte("خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active imperfective preterite indicative - { - input: []byte("مي خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active future indicative - { - input: []byte("خواهد خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active present progressive indicative - { - input: []byte("دارد مي خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active preterite progressive indicative - { - input: []byte("داشت مي خورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورد"), - }, - }, - }, - // active perfect indicative - { - input: []byte("خورده است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective perfect indicative - { - input: []byte("مي خورده است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active pluperfect indicative - { - input: []byte("خورده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective pluperfect indicative - { - input: []byte("مي خورده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active preterite subjunctive - { - input: []byte("خورده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective preterite subjunctive - { - input: []byte("مي خورده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active pluperfect subjunctive - { - input: []byte("خورده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active imperfective pluperfect subjunctive - { - input: []byte("مي خورده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive present indicative - { - input: []byte("خورده مي شود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive preterite indicative - { - input: []byte("خورده شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective preterite indicative - { - input: []byte("خورده مي شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive perfect indicative - { - input: []byte("خورده شده است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective perfect indicative - { - input: []byte("خورده مي شده است"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive pluperfect indicative - { - input: []byte("خورده شده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective pluperfect indicative - { - input: []byte("خورده مي شده بود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive future indicative - { - input: []byte("خورده خواهد شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive present progressive indicative - { - input: []byte("دارد خورده مي شود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive preterite progressive indicative - { - input: []byte("داشت خورده مي شد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive present subjunctive - { - input: []byte("خورده شود"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive preterite subjunctive - { - input: []byte("خورده شده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective preterite subjunctive - { - input: []byte("خورده مي شده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive pluperfect subjunctive - { - input: []byte("خورده شده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // passive imperfective pluperfect subjunctive - { - input: []byte("خورده مي شده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - // active present subjunctive - { - input: []byte("بخورد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("بخورد"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} - -func TestPersianAnalyzerOthers(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // nouns - { - input: []byte("برگ ها"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("برگ"), - }, - }, - }, - { - input: []byte("برگ‌ها"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("برگ"), - }, - }, - }, - // non persian - { - input: []byte("English test."), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("english"), - }, - &analysis.Token{ - Term: []byte("test"), - }, - }, - }, - // others - { - input: []byte("خورده مي شده بوده باشد"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("خورده"), - }, - }, - }, - { - input: []byte("برگ‌ها"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("برگ"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize.go b/vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize.go deleted file mode 100644 index ab67af3a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fa - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const NormalizeName = "normalize_fa" - -const ( - Yeh = '\u064A' - FarsiYeh = '\u06CC' - YehBarree = '\u06D2' - Keheh = '\u06A9' - Kaf = '\u0643' - HamzaAbove = '\u0654' - HehYeh = '\u06C0' - HehGoal = '\u06C1' - Heh = '\u0647' -) - -type PersianNormalizeFilter struct { -} - -func NewPersianNormalizeFilter() *PersianNormalizeFilter { - return &PersianNormalizeFilter{} -} - -func (s *PersianNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - term := normalize(token.Term) - token.Term = term - } - return input -} - -func normalize(input []byte) []byte { - runes := bytes.Runes(input) - for i := 0; i < len(runes); i++ { - switch runes[i] { - case FarsiYeh, YehBarree: - runes[i] = Yeh - case Keheh: - runes[i] = Kaf - case HehYeh, HehGoal: - runes[i] = Heh - case HamzaAbove: // necessary for HEH + HAMZA - runes = analysis.DeleteRune(runes, i) - i-- - } - } - return analysis.BuildTermFromRunes(runes) -} - -func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewPersianNormalizeFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize_test.go deleted file mode 100644 index 7c662958..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fa/persian_normalize_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fa - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestPersianNormalizeFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // FarsiYeh - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("های"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هاي"), - }, - }, - }, - // YehBarree - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هاے"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("هاي"), - }, - }, - }, - // Keheh - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("کشاندن"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كشاندن"), - }, - }, - }, - // HehYeh - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كتابۀ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كتابه"), - }, - }, - }, - // HehHamzaAbove - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كتابهٔ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("كتابه"), - }, - }, - }, - // HehGoal - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("زادہ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("زاده"), - }, - }, - }, - // empty - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - persianNormalizeFilter := NewPersianNormalizeFilter() - for _, test := range tests { - actual := persianNormalizeFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_filter_fa.go b/vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_filter_fa.go deleted file mode 100644 index bda7d666..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_filter_fa.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fa - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_words_fa.go b/vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_words_fa.go deleted file mode 100644 index 0d8ce409..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fa/stop_words_fa.go +++ /dev/null @@ -1,337 +0,0 @@ -package fa - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_fa" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var PersianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -# Note: by default this file is used after normalization, so when adding entries -# to this file, use the arabic 'ي' instead of 'ی' -انان -نداشته -سراسر -خياه -ايشان -وي -تاكنون -بيشتري -دوم -پس -ناشي -وگو -يا -داشتند -سپس -هنگام -هرگز -پنج -نشان -امسال -ديگر -گروهي -شدند -چطور -ده -و -دو -نخستين -ولي -چرا -چه -وسط -ه -كدام -قابل -يك -رفت -هفت -همچنين -در -هزار -بله -بلي -شايد -اما -شناسي -گرفته -دهد -داشته -دانست -داشتن -خواهيم -ميليارد -وقتيكه -امد -خواهد -جز -اورده -شده -بلكه -خدمات -شدن -برخي -نبود -بسياري -جلوگيري -حق -كردند -نوعي -بعري -نكرده -نظير -نبايد -بوده -بودن -داد -اورد -هست -جايي -شود -دنبال -داده -بايد -سابق -هيچ -همان -انجا -كمتر -كجاست -گردد -كسي -تر -مردم -تان -دادن -بودند -سري -جدا -ندارند -مگر -يكديگر -دارد -دهند -بنابراين -هنگامي -سمت -جا -انچه -خود -دادند -زياد -دارند -اثر -بدون -بهترين -بيشتر -البته -به -براساس -بيرون -كرد -بعضي -گرفت -توي -اي -ميليون -او -جريان -تول -بر -مانند -برابر -باشيم -مدتي -گويند -اكنون -تا -تنها -جديد -چند -بي -نشده -كردن -كردم -گويد -كرده -كنيم -نمي -نزد -روي -قصد -فقط -بالاي -ديگران -اين -ديروز -توسط -سوم -ايم -دانند -سوي -استفاده -شما -كنار -داريم -ساخته -طور -امده -رفته -نخست -بيست -نزديك -طي -كنيد -از -انها -تمامي -داشت -يكي -طريق -اش -چيست -روب -نمايد -گفت -چندين -چيزي -تواند -ام -ايا -با -ان -ايد -ترين -اينكه -ديگري -راه -هايي -بروز -همچنان -پاعين -كس -حدود -مختلف -مقابل -چيز -گيرد -ندارد -ضد -همچون -سازي -شان -مورد -باره -مرسي -خويش -برخوردار -چون -خارج -شش -هنوز -تحت -ضمن -هستيم -گفته -فكر -بسيار -پيش -براي -روزهاي -انكه -نخواهد -بالا -كل -وقتي -كي -چنين -كه -گيري -نيست -است -كجا -كند -نيز -يابد -بندي -حتي -توانند -عقب -خواست -كنند -بين -تمام -همه -ما -باشند -مثل -شد -اري -باشد -اره -طبق -بعد -اگر -صورت -غير -جاي -بيش -ريزي -اند -زيرا -چگونه -بار -لطفا -مي -درباره -من -ديده -همين -گذاري -برداري -علت -گذاشته -هم -فوق -نه -ها -شوند -اباد -همواره -هر -اول -خواهند -چهار -نام -امروز -مان -هاي -قبل -كنم -سعي -تازه -را -هستند -زير -جلوي -عنوان -بود -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(PersianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi.go b/vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi.go deleted file mode 100644 index 909c54e5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package fi - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "fi" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopFiFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerFiFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopFiFilter, - stemmerFiFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi_test.go deleted file mode 100644 index faaa03dd..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fi/analyzer_fi_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package fi - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestFinishAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("edeltäjiinsä"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("edeltäj"), - }, - }, - }, - { - input: []byte("edeltäjistään"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("edeltäj"), - }, - }, - }, - // stop word - { - input: []byte("olla"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fi/stemmer_fi.go b/vendor/github.com/blevesearch/bleve/analysis/language/fi/stemmer_fi.go deleted file mode 100644 index 454647b5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fi/stemmer_fi.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package fi - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_fi" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("fi") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_filter_fi.go b/vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_filter_fi.go deleted file mode 100644 index 3c5a2f0d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_filter_fi.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fi - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_words_fi.go b/vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_words_fi.go deleted file mode 100644 index 7cf0c9c1..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fi/stop_words_fi.go +++ /dev/null @@ -1,121 +0,0 @@ -package fi - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_fi" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var FinnishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - -| forms of BE - -olla -olen -olet -on -olemme -olette -ovat -ole | negative form - -oli -olisi -olisit -olisin -olisimme -olisitte -olisivat -olit -olin -olimme -olitte -olivat -ollut -olleet - -en | negation -et -ei -emme -ette -eivät - -|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans -minä minun minut minua minussa minusta minuun minulla minulta minulle | I -sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you -hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she -me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we -te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you -he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they - -tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this -tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that -se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it -nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these -nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those -ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they - -kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who -ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) -mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what -mitkä | (pl) - -joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which -jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) - -| conjunctions - -että | that -ja | and -jos | if -koska | because -kuin | than -mutta | but -niin | so -sekä | and -sillä | for -tai | or -vaan | but -vai | or -vaikka | although - - -| prepositions - -kanssa | with -mukaan | according to -noin | about -poikki | across -yli | over, across - -| other - -kun | when -niin | so -nyt | now -itse | self - -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(FinnishStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr.go deleted file mode 100644 index 6c6f60f3..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" -) - -const AnalyzerName = "fr" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - elisionFilter, err := cache.TokenFilterNamed(ElisionName) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopFrFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerFrFilter, err := cache.TokenFilterNamed(LightStemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - elisionFilter, - toLowerFilter, - stopFrFilter, - stemmerFrFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr_test.go deleted file mode 100644 index cfb500b5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/analyzer_fr_test.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestFrenchAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - input: []byte(""), - output: analysis.TokenStream{}, - }, - { - input: []byte("chien chat cheval"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chien"), - }, - &analysis.Token{ - Term: []byte("chat"), - }, - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - }, - { - input: []byte("chien CHAT CHEVAL"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chien"), - }, - &analysis.Token{ - Term: []byte("chat"), - }, - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - }, - { - input: []byte(" chien ,? + = - CHAT /: > CHEVAL"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chien"), - }, - &analysis.Token{ - Term: []byte("chat"), - }, - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - }, - { - input: []byte("chien++"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chien"), - }, - }, - }, - { - input: []byte("mot \"entreguillemet\""), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("mot"), - }, - &analysis.Token{ - Term: []byte("entreguilemet"), - }, - }, - }, - { - input: []byte("Jean-François"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("jean"), - }, - &analysis.Token{ - Term: []byte("francoi"), - }, - }, - }, - // stop words - { - input: []byte("le la chien les aux chat du des à cheval"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chien"), - }, - &analysis.Token{ - Term: []byte("chat"), - }, - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - }, - // nouns and adjectives - { - input: []byte("lances chismes habitable chiste éléments captifs"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("lanc"), - }, - &analysis.Token{ - Term: []byte("chism"), - }, - &analysis.Token{ - Term: []byte("habitabl"), - }, - &analysis.Token{ - Term: []byte("chist"), - }, - &analysis.Token{ - Term: []byte("element"), - }, - &analysis.Token{ - Term: []byte("captif"), - }, - }, - }, - // verbs - { - input: []byte("finissions souffrirent rugissante"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("finision"), - }, - &analysis.Token{ - Term: []byte("soufrirent"), - }, - &analysis.Token{ - Term: []byte("rugisant"), - }, - }, - }, - { - input: []byte("C3PO aujourd'hui oeuf ïâöûàä anticonstitutionnellement Java++ "), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("c3po"), - }, - &analysis.Token{ - Term: []byte("aujourd'hui"), - }, - &analysis.Token{ - Term: []byte("oeuf"), - }, - &analysis.Token{ - Term: []byte("ïaöuaä"), - }, - &analysis.Token{ - Term: []byte("anticonstitutionel"), - }, - &analysis.Token{ - Term: []byte("java"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/articles_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/articles_fr.go deleted file mode 100644 index 557409a2..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/articles_fr.go +++ /dev/null @@ -1,37 +0,0 @@ -package fr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const ArticlesName = "articles_fr" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis - -var FrenchArticles = []byte(` -l -m -t -qu -n -s -j -d -c -jusqu -quoiqu -lorsqu -puisqu -`) - -func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(FrenchArticles) - return rv, err -} - -func init() { - registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr.go deleted file mode 100644 index b8b20ab5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/elision_filter" - "github.com/blevesearch/bleve/registry" -) - -const ElisionName = "elision_fr" - -func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) - if err != nil { - return nil, fmt.Errorf("error building elision filter: %v", err) - } - return elision_filter.NewElisionFilter(articlesTokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr_test.go deleted file mode 100644 index a3bb0512..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/elision_fr_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestFrenchElision(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("l'avion"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("avion"), - }, - }, - }, - } - - cache := registry.NewCache() - elisionFilter, err := cache.TokenFilterNamed(ElisionName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := elisionFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr.go deleted file mode 100644 index 1b598a3e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "bytes" - "unicode" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const LightStemmerName = "stemmer_fr_light" - -type FrenchLightStemmerFilter struct { -} - -func NewFrenchLightStemmerFilter() *FrenchLightStemmerFilter { - return &FrenchLightStemmerFilter{} -} - -func (s *FrenchLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - runes := bytes.Runes(token.Term) - runes = stem(runes) - token.Term = analysis.BuildTermFromRunes(runes) - } - return input -} - -func stem(input []rune) []rune { - - inputLen := len(input) - - if inputLen > 5 && input[inputLen-1] == 'x' { - if input[inputLen-3] == 'a' && input[inputLen-2] == 'u' && input[inputLen-4] != 'e' { - input[inputLen-2] = 'l' - } - input = input[0 : inputLen-1] - inputLen = len(input) - } - - if inputLen > 3 && input[inputLen-1] == 'x' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - - if inputLen > 3 && input[inputLen-1] == 's' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - - if inputLen > 9 && analysis.RunesEndsWith(input, "issement") { - input = input[0 : inputLen-6] - inputLen = len(input) - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "issant") { - input = input[0 : inputLen-4] - inputLen = len(input) - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 6 && analysis.RunesEndsWith(input, "ement") { - input = input[0 : inputLen-4] - inputLen = len(input) - if inputLen > 3 && analysis.RunesEndsWith(input, "ive") { - input = input[0 : inputLen-1] - inputLen = len(input) - input[inputLen-1] = 'f' - } - return norm(input) - } - - if inputLen > 11 && analysis.RunesEndsWith(input, "ficatrice") { - input = input[0 : inputLen-5] - inputLen = len(input) - input[inputLen-2] = 'e' - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 10 && analysis.RunesEndsWith(input, "ficateur") { - input = input[0 : inputLen-4] - inputLen = len(input) - input[inputLen-2] = 'e' - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 9 && analysis.RunesEndsWith(input, "catrice") { - input = input[0 : inputLen-3] - inputLen = len(input) - input[inputLen-4] = 'q' - input[inputLen-3] = 'u' - input[inputLen-2] = 'e' - //s[len-1] = 'r' <-- unnecessary, already 'r'. - return norm(input) - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "cateur") { - input = input[0 : inputLen-2] - inputLen = len(input) - input[inputLen-4] = 'q' - input[inputLen-3] = 'u' - input[inputLen-2] = 'e' - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "atrice") { - input = input[0 : inputLen-4] - inputLen = len(input) - input[inputLen-2] = 'e' - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 7 && analysis.RunesEndsWith(input, "ateur") { - input = input[0 : inputLen-3] - inputLen = len(input) - input[inputLen-2] = 'e' - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 6 && analysis.RunesEndsWith(input, "trice") { - input = input[0 : inputLen-1] - inputLen = len(input) - input[inputLen-3] = 'e' - input[inputLen-2] = 'u' - input[inputLen-1] = 'r' - } - - if inputLen > 5 && analysis.RunesEndsWith(input, "ième") { - return norm(input[0 : inputLen-4]) - } - - if inputLen > 7 && analysis.RunesEndsWith(input, "teuse") { - input = input[0 : inputLen-2] - inputLen = len(input) - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 6 && analysis.RunesEndsWith(input, "teur") { - input = input[0 : inputLen-1] - inputLen = len(input) - input[inputLen-1] = 'r' - return norm(input) - } - - if inputLen > 5 && analysis.RunesEndsWith(input, "euse") { - return norm(input[0 : inputLen-2]) - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "ère") { - input = input[0 : inputLen-1] - inputLen = len(input) - input[inputLen-2] = 'e' - return norm(input) - } - - if inputLen > 7 && analysis.RunesEndsWith(input, "ive") { - input = input[0 : inputLen-1] - inputLen = len(input) - input[inputLen-1] = 'f' - return norm(input) - } - - if inputLen > 4 && - (analysis.RunesEndsWith(input, "folle") || - analysis.RunesEndsWith(input, "molle")) { - input = input[0 : inputLen-2] - inputLen = len(input) - input[inputLen-1] = 'u' - return norm(input) - } - - if inputLen > 9 && analysis.RunesEndsWith(input, "nnelle") { - return norm(input[0 : inputLen-5]) - } - - if inputLen > 9 && analysis.RunesEndsWith(input, "nnel") { - return norm(input[0 : inputLen-3]) - } - - if inputLen > 4 && analysis.RunesEndsWith(input, "ète") { - input = input[0 : inputLen-1] - inputLen = len(input) - input[inputLen-2] = 'e' - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "ique") { - input = input[0 : inputLen-4] - inputLen = len(input) - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "esse") { - return norm(input[0 : inputLen-3]) - } - - if inputLen > 7 && analysis.RunesEndsWith(input, "inage") { - return norm(input[0 : inputLen-3]) - } - - if inputLen > 9 && analysis.RunesEndsWith(input, "isation") { - input = input[0 : inputLen-7] - inputLen = len(input) - if inputLen > 5 && analysis.RunesEndsWith(input, "ual") { - input[inputLen-2] = 'e' - } - return norm(input) - } - - if inputLen > 9 && analysis.RunesEndsWith(input, "isateur") { - return norm(input[0 : inputLen-7]) - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "ation") { - return norm(input[0 : inputLen-5]) - } - - if inputLen > 8 && analysis.RunesEndsWith(input, "ition") { - return norm(input[0 : inputLen-5]) - } - - return norm(input) - -} - -func norm(input []rune) []rune { - - inputLen := len(input) - if inputLen > 4 { - for i := 0; i < inputLen; i++ { - switch input[i] { - case 'à', 'á', 'â': - input[i] = 'a' - case 'ô': - input[i] = 'o' - case 'è', 'é', 'ê': - input[i] = 'e' - case 'ù', 'û': - input[i] = 'u' - case 'î': - input[i] = 'i' - case 'ç': - input[i] = 'c' - } - - ch := input[0] - for i := 1; i < inputLen; i++ { - if input[i] == ch && unicode.IsLetter(ch) { - input = analysis.DeleteRune(input, i) - i -= 1 - inputLen = len(input) - } else { - ch = input[i] - } - } - } - } - - if inputLen > 4 && analysis.RunesEndsWith(input, "ie") { - input = input[0 : inputLen-2] - inputLen = len(input) - } - - if inputLen > 4 { - if input[inputLen-1] == 'r' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - if input[inputLen-1] == 'e' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - if input[inputLen-1] == 'e' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - if input[inputLen-1] == input[inputLen-2] && unicode.IsLetter(input[inputLen-1]) { - input = input[0 : inputLen-1] - inputLen = len(input) - } - } - - return input -} - -func FrenchLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewFrenchLightStemmerFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(LightStemmerName, FrenchLightStemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr_test.go deleted file mode 100644 index 915e730e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/light_stemmer_fr_test.go +++ /dev/null @@ -1,997 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestFrenchLightStemmer(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chevaux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("hiboux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("hibou"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("hibou"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("hibou"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chantés"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chanter"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chante"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baronnes"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("barons"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("peaux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("peau"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("peau"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("peau"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("anneaux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("aneau"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("anneau"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("aneau"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("neveux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("neveu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("neveu"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("neveu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("affreux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("afreu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("affreuse"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("afreu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("investissement"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("investi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("investir"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("investi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("assourdissant"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("asourdi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("assourdir"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("asourdi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("pratiquement"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("pratiqu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("pratique"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("pratiqu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administrativement"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administratif"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administratif"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administratif"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("justificatrice"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("justifi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("justificateur"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("justifi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("justifier"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("justifi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("educatrice"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("eduqu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("eduquer"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("eduqu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("communicateur"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("comuniqu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("communiquer"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("comuniqu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("accompagnatrice"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("acompagn"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("accompagnateur"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("acompagn"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administrateur"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administr"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administrer"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("administr"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("productrice"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("product"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("producteur"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("product"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("acheteuse"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("achet"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("acheteur"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("achet"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("planteur"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("plant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("plante"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("plant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("poreuse"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("poreu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("poreux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("poreu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("plieuse"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("plieu"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("bijoutière"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("bijouti"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("bijoutier"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("bijouti"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("caissière"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("caisi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("caissier"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("caisi"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abrasive"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abrasif"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abrasif"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abrasif"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("folle"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("fou"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("fou"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("fou"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("personnelle"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("person"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("personne"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("person"), - }, - }, - }, - // algo bug: too short length - // { - // input: analysis.TokenStream{ - // &analysis.Token{ - // Term: []byte("personnel"), - // }, - // }, - // output: analysis.TokenStream{ - // &analysis.Token{ - // Term: []byte("person"), - // }, - // }, - // }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("complète"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("complet"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("complet"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("complet"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("aromatique"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("aromat"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("faiblesse"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("faibl"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("faible"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("faibl"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("patinage"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("patin"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("patin"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("patin"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("sonorisation"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("sono"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ritualisation"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("rituel"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("rituel"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("rituel"), - }, - }, - }, - // algo bug: masked by rules above - // { - // input: analysis.TokenStream{ - // &analysis.Token{ - // Term: []byte("colonisateur"), - // }, - // }, - // output: analysis.TokenStream{ - // &analysis.Token{ - // Term: []byte("colon"), - // }, - // }, - // }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("nomination"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("nomin"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("disposition"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("dispos"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("dispose"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("dispos"), - }, - }, - }, - // SOLR-3463 : abusive compression of repeated characters in numbers - // Trailing repeated char elision : - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1234555"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1234555"), - }, - }, - }, - // Repeated char within numbers with more than 4 characters : - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("12333345"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("12333345"), - }, - }, - }, - // Short numbers weren't affected already: - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1234"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1234"), - }, - }, - }, - // Ensure behaviour is preserved for words! - // Trailing repeated char elision : - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcdeff"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcdef"), - }, - }, - }, - // Repeated char within words with more than 4 characters : - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcccddeef"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcdef"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("créées"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("cre"), - }, - }, - }, - // Combined letter and digit repetition - // 10:00pm - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("22hh00"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("22h00"), - }, - }, - }, - } - - cache := registry.NewCache() - filter, err := cache.TokenFilterNamed(LightStemmerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := filter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr.go deleted file mode 100644 index 4e09b90f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const MinimalStemmerName = "stemmer_fr_min" - -type FrenchMinimalStemmerFilter struct { -} - -func NewFrenchMinimalStemmerFilter() *FrenchMinimalStemmerFilter { - return &FrenchMinimalStemmerFilter{} -} - -func (s *FrenchMinimalStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - runes := bytes.Runes(token.Term) - runes = minstem(runes) - token.Term = analysis.BuildTermFromRunes(runes) - } - return input -} - -func minstem(input []rune) []rune { - - inputLen := len(input) - - if inputLen < 6 { - return input - } - - if input[inputLen-1] == 'x' { - if input[inputLen-3] == 'a' && input[inputLen-2] == 'u' { - input[inputLen-2] = 'l' - } - return input[0 : inputLen-1] - } - - if input[inputLen-1] == 's' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - if input[inputLen-1] == 'r' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - if input[inputLen-1] == 'e' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - if input[inputLen-1] == 'é' { - input = input[0 : inputLen-1] - inputLen = len(input) - } - if input[inputLen-1] == input[inputLen-2] { - input = input[0 : inputLen-1] - inputLen = len(input) - } - return input -} - -func FrenchMinimalStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewFrenchMinimalStemmerFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(MinimalStemmerName, FrenchMinimalStemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr_test.go deleted file mode 100644 index b7aa0ffe..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/minimal_stemmer_fr_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestFrenchMinimalStemmer(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chevaux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("cheval"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("hiboux"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("hibou"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chantés"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chanter"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chante"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("chant"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baronnes"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("barons"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("baron"), - }, - }, - }, - } - - cache := registry.NewCache() - filter, err := cache.TokenFilterNamed(MinimalStemmerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := filter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/stemmer_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/stemmer_fr.go deleted file mode 100644 index a5edccb7..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/stemmer_fr.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package fr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_fr" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("fr") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_filter_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_filter_fr.go deleted file mode 100644 index d5233e01..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_filter_fr.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package fr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_words_fr.go b/vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_words_fr.go deleted file mode 100644 index f00266f0..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/fr/stop_words_fr.go +++ /dev/null @@ -1,210 +0,0 @@ -package fr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_fr" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var FrenchStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A French stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -au | a + le -aux | a + les -avec | with -ce | this -ces | these -dans | with -de | of -des | de + les -du | de + le -elle | she -en | 'of them' etc -et | and -eux | them -il | he -je | I -la | the -le | the -leur | their -lui | him -ma | my (fem) -mais | but -me | me -même | same; as in moi-même (myself) etc -mes | me (pl) -moi | me -mon | my (masc) -ne | not -nos | our (pl) -notre | our -nous | we -on | one -ou | where -par | by -pas | not -pour | for -qu | que before vowel -que | that -qui | who -sa | his, her (fem) -se | oneself -ses | his (pl) -son | his, her (masc) -sur | on -ta | thy (fem) -te | thee -tes | thy (pl) -toi | thee -ton | thy (masc) -tu | thou -un | a -une | a -vos | your (pl) -votre | your -vous | you - - | single letter forms - -c | c' -d | d' -j | j' -l | l' -à | to, at -m | m' -n | n' -s | s' -t | t' -y | there - - | forms of être (not including the infinitive): -été -étée -étées -étés -étant -suis -es -est -sommes -êtes -sont -serai -seras -sera -serons -serez -seront -serais -serait -serions -seriez -seraient -étais -était -étions -étiez -étaient -fus -fut -fûmes -fûtes -furent -sois -soit -soyons -soyez -soient -fusse -fusses -fût -fussions -fussiez -fussent - - | forms of avoir (not including the infinitive): -ayant -eu -eue -eues -eus -ai -as -avons -avez -ont -aurai -auras -aura -aurons -aurez -auront -aurais -aurait -aurions -auriez -auraient -avais -avait -avions -aviez -avaient -eut -eûmes -eûtes -eurent -aie -aies -ait -ayons -ayez -aient -eusse -eusses -eût -eussions -eussiez -eussent - - | Later additions (from Jean-Christophe Deschamps) -ceci | this -cela | that -celà | that -cet | this -cette | this -ici | here -ils | they -les | the (pl) -leurs | their (pl) -quel | which -quels | which -quelle | which -quelles | which -sans | without -soi | oneself - -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(FrenchStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ga/articles_ga.go b/vendor/github.com/blevesearch/bleve/analysis/language/ga/articles_ga.go deleted file mode 100644 index 0b6863a6..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ga/articles_ga.go +++ /dev/null @@ -1,27 +0,0 @@ -package ga - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const ArticlesName = "articles_ga" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis - -var IrishArticles = []byte(` -d -m -b -`) - -func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(IrishArticles) - return rv, err -} - -func init() { - registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga.go b/vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga.go deleted file mode 100644 index c6aeac2c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ga - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/elision_filter" - "github.com/blevesearch/bleve/registry" -) - -const ElisionName = "elision_ga" - -func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) - if err != nil { - return nil, fmt.Errorf("error building elision filter: %v", err) - } - return elision_filter.NewElisionFilter(articlesTokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga_test.go deleted file mode 100644 index ae667acd..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ga/elision_ga_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ga - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestFrenchElision(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("b'fhearr"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("fhearr"), - }, - }, - }, - } - - cache := registry.NewCache() - elisionFilter, err := cache.TokenFilterNamed(ElisionName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := elisionFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_filter_ga.go b/vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_filter_ga.go deleted file mode 100644 index 43bf4819..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_filter_ga.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ga - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_words_ga.go b/vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_words_ga.go deleted file mode 100644 index 12703be0..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ga/stop_words_ga.go +++ /dev/null @@ -1,134 +0,0 @@ -package ga - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_ga" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var IrishStopWords = []byte(` -a -ach -ag -agus -an -aon -ar -arna -as -b' -ba -beirt -bhúr -caoga -ceathair -ceathrar -chomh -chtó -chuig -chun -cois -céad -cúig -cúigear -d' -daichead -dar -de -deich -deichniúr -den -dhá -do -don -dtí -dá -dár -dó -faoi -faoin -faoina -faoinár -fara -fiche -gach -gan -go -gur -haon -hocht -i -iad -idir -in -ina -ins -inár -is -le -leis -lena -lenár -m' -mar -mo -mé -na -nach -naoi -naonúr -ná -ní -níor -nó -nócha -ocht -ochtar -os -roimh -sa -seacht -seachtar -seachtó -seasca -seisear -siad -sibh -sinn -sna -sé -sí -tar -thar -thú -triúr -trí -trína -trínár -tríocha -tú -um -ár -é -éis -í -ó -ón -óna -ónár -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(IrishStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_filter_gl.go b/vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_filter_gl.go deleted file mode 100644 index bc6e58ab..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_filter_gl.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package gl - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_words_gl.go b/vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_words_gl.go deleted file mode 100644 index 82a551c4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/gl/stop_words_gl.go +++ /dev/null @@ -1,185 +0,0 @@ -package gl - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_gl" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var GalicianStopWords = []byte(`# galican stopwords -a -aínda -alí -aquel -aquela -aquelas -aqueles -aquilo -aquí -ao -aos -as -así -á -ben -cando -che -co -coa -comigo -con -connosco -contigo -convosco -coas -cos -cun -cuns -cunha -cunhas -da -dalgunha -dalgunhas -dalgún -dalgúns -das -de -del -dela -delas -deles -desde -deste -do -dos -dun -duns -dunha -dunhas -e -el -ela -elas -eles -en -era -eran -esa -esas -ese -eses -esta -estar -estaba -está -están -este -estes -estiven -estou -eu -é -facer -foi -foron -fun -había -hai -iso -isto -la -las -lle -lles -lo -los -mais -me -meu -meus -min -miña -miñas -moi -na -nas -neste -nin -no -non -nos -nosa -nosas -noso -nosos -nós -nun -nunha -nuns -nunhas -o -os -ou -ó -ós -para -pero -pode -pois -pola -polas -polo -polos -por -que -se -senón -ser -seu -seus -sexa -sido -sobre -súa -súas -tamén -tan -te -ten -teñen -teño -ter -teu -teus -ti -tido -tiña -tiven -túa -túas -un -unha -unhas -uns -vos -vosa -vosas -voso -vosos -vós -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(GalicianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi.go deleted file mode 100644 index 00b40a0a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hi - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/language/in" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" -) - -const AnalyzerName = "hi" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - indicNormalizeFilter, err := cache.TokenFilterNamed(in.NormalizeName) - if err != nil { - return nil, err - } - hindiNormalizeFilter, err := cache.TokenFilterNamed(NormalizeName) - if err != nil { - return nil, err - } - stopHiFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerHiFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - indicNormalizeFilter, - hindiNormalizeFilter, - stopHiFilter, - stemmerHiFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi_test.go deleted file mode 100644 index 04058362..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/analyzer_hi_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hi - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestHindiAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // two ways to write 'hindi' itself - { - input: []byte("हिन्दी"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("हिंद"), - Position: 1, - Start: 0, - End: 18, - }, - }, - }, - { - input: []byte("हिंदी"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("हिंद"), - Position: 1, - Start: 0, - End: 15, - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize.go deleted file mode 100644 index f9b607c6..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hi - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const NormalizeName = "normalize_hi" - -type HindiNormalizeFilter struct { -} - -func NewHindiNormalizeFilter() *HindiNormalizeFilter { - return &HindiNormalizeFilter{} -} - -func (s *HindiNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - term := normalize(token.Term) - token.Term = term - } - return input -} - -func normalize(input []byte) []byte { - runes := bytes.Runes(input) - for i := 0; i < len(runes); i++ { - switch runes[i] { - // dead n -> bindu - case '\u0928': - if i+1 < len(runes) && runes[i+1] == '\u094D' { - runes[i] = '\u0902' - runes = analysis.DeleteRune(runes, i+1) - } - // candrabindu -> bindu - case '\u0901': - runes[i] = '\u0902' - // nukta deletions - case '\u093C': - runes = analysis.DeleteRune(runes, i) - i-- - case '\u0929': - runes[i] = '\u0928' - case '\u0931': - runes[i] = '\u0930' - case '\u0934': - runes[i] = '\u0933' - case '\u0958': - runes[i] = '\u0915' - case '\u0959': - runes[i] = '\u0916' - case '\u095A': - runes[i] = '\u0917' - case '\u095B': - runes[i] = '\u091C' - case '\u095C': - runes[i] = '\u0921' - case '\u095D': - runes[i] = '\u0922' - case '\u095E': - runes[i] = '\u092B' - case '\u095F': - runes[i] = '\u092F' - // zwj/zwnj -> delete - case '\u200D', '\u200C': - runes = analysis.DeleteRune(runes, i) - i-- - // virama -> delete - case '\u094D': - runes = analysis.DeleteRune(runes, i) - i-- - // chandra/short -> replace - case '\u0945', '\u0946': - runes[i] = '\u0947' - case '\u0949', '\u094A': - runes[i] = '\u094B' - case '\u090D', '\u090E': - runes[i] = '\u090F' - case '\u0911', '\u0912': - runes[i] = '\u0913' - case '\u0972': - runes[i] = '\u0905' - // long -> short ind. vowels - case '\u0906': - runes[i] = '\u0905' - case '\u0908': - runes[i] = '\u0907' - case '\u090A': - runes[i] = '\u0909' - case '\u0960': - runes[i] = '\u090B' - case '\u0961': - runes[i] = '\u090C' - case '\u0910': - runes[i] = '\u090F' - case '\u0914': - runes[i] = '\u0913' - // long -> short dep. vowels - case '\u0940': - runes[i] = '\u093F' - case '\u0942': - runes[i] = '\u0941' - case '\u0944': - runes[i] = '\u0943' - case '\u0963': - runes[i] = '\u0962' - case '\u0948': - runes[i] = '\u0947' - case '\u094C': - runes[i] = '\u094B' - } - } - return analysis.BuildTermFromRunes(runes) -} - -func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewHindiNormalizeFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize_test.go deleted file mode 100644 index 4c8114c9..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_normalize_test.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hi - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestHindiNormalizeFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // basics - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अँगरेज़ी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अँगरेजी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अँग्रेज़ी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अँग्रेजी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेज़ी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंग्रेज़ी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंग्रेजी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अंगरेजि"), - }, - }, - }, - // test decompositions - // removing nukta dot - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("क़िताब"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("किताब"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("फ़र्ज़"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("फरज"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("क़र्ज़"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("करज"), - }, - }, - }, - // some other composed nukta forms - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ऱऴख़ग़ड़ढ़य़"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("रळखगडढय"), - }, - }, - }, - // removal of format (ZWJ/ZWNJ) - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("शार्‍मा"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("शारमा"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("शार्‌मा"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("शारमा"), - }, - }, - }, - // removal of chandra - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ॅॆॉॊऍऎऑऒ\u0972"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ेेोोएएओओअ"), - }, - }, - }, - // vowel shortening - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आईऊॠॡऐऔीूॄॣैौ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अइउऋऌएओिुृॢेो"), - }, - }, - }, - // empty - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - hindiNormalizeFilter := NewHindiNormalizeFilter() - for _, test := range tests { - actual := hindiNormalizeFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter.go deleted file mode 100644 index b4e6ddd6..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hi - -import ( - "bytes" - "unicode/utf8" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_hi" - -type HindiStemmerFilter struct { -} - -func NewHindiStemmerFilter() *HindiStemmerFilter { - return &HindiStemmerFilter{} -} - -func (s *HindiStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - // if not protected keyword, stem it - if !token.KeyWord { - stemmed := stem(token.Term) - token.Term = stemmed - } - } - return input -} - -func stem(input []byte) []byte { - inputLen := utf8.RuneCount(input) - - // 5 - if inputLen > 6 && - (bytes.HasSuffix(input, []byte("ाएंगी")) || - bytes.HasSuffix(input, []byte("ाएंगे")) || - bytes.HasSuffix(input, []byte("ाऊंगी")) || - bytes.HasSuffix(input, []byte("ाऊंगा")) || - bytes.HasSuffix(input, []byte("ाइयाँ")) || - bytes.HasSuffix(input, []byte("ाइयों")) || - bytes.HasSuffix(input, []byte("ाइयां"))) { - return analysis.TruncateRunes(input, 5) - } - - // 4 - if inputLen > 5 && - (bytes.HasSuffix(input, []byte("ाएगी")) || - bytes.HasSuffix(input, []byte("ाएगा")) || - bytes.HasSuffix(input, []byte("ाओगी")) || - bytes.HasSuffix(input, []byte("ाओगे")) || - bytes.HasSuffix(input, []byte("एंगी")) || - bytes.HasSuffix(input, []byte("ेंगी")) || - bytes.HasSuffix(input, []byte("एंगे")) || - bytes.HasSuffix(input, []byte("ेंगे")) || - bytes.HasSuffix(input, []byte("ूंगी")) || - bytes.HasSuffix(input, []byte("ूंगा")) || - bytes.HasSuffix(input, []byte("ातीं")) || - bytes.HasSuffix(input, []byte("नाओं")) || - bytes.HasSuffix(input, []byte("नाएं")) || - bytes.HasSuffix(input, []byte("ताओं")) || - bytes.HasSuffix(input, []byte("ताएं")) || - bytes.HasSuffix(input, []byte("ियाँ")) || - bytes.HasSuffix(input, []byte("ियों")) || - bytes.HasSuffix(input, []byte("ियां"))) { - return analysis.TruncateRunes(input, 4) - } - - // 3 - if inputLen > 4 && - (bytes.HasSuffix(input, []byte("ाकर")) || - bytes.HasSuffix(input, []byte("ाइए")) || - bytes.HasSuffix(input, []byte("ाईं")) || - bytes.HasSuffix(input, []byte("ाया")) || - bytes.HasSuffix(input, []byte("ेगी")) || - bytes.HasSuffix(input, []byte("ेगा")) || - bytes.HasSuffix(input, []byte("ोगी")) || - bytes.HasSuffix(input, []byte("ोगे")) || - bytes.HasSuffix(input, []byte("ाने")) || - bytes.HasSuffix(input, []byte("ाना")) || - bytes.HasSuffix(input, []byte("ाते")) || - bytes.HasSuffix(input, []byte("ाती")) || - bytes.HasSuffix(input, []byte("ाता")) || - bytes.HasSuffix(input, []byte("तीं")) || - bytes.HasSuffix(input, []byte("ाओं")) || - bytes.HasSuffix(input, []byte("ाएं")) || - bytes.HasSuffix(input, []byte("ुओं")) || - bytes.HasSuffix(input, []byte("ुएं")) || - bytes.HasSuffix(input, []byte("ुआं"))) { - return analysis.TruncateRunes(input, 3) - } - - // 2 - if inputLen > 3 && - (bytes.HasSuffix(input, []byte("कर")) || - bytes.HasSuffix(input, []byte("ाओ")) || - bytes.HasSuffix(input, []byte("िए")) || - bytes.HasSuffix(input, []byte("ाई")) || - bytes.HasSuffix(input, []byte("ाए")) || - bytes.HasSuffix(input, []byte("ने")) || - bytes.HasSuffix(input, []byte("नी")) || - bytes.HasSuffix(input, []byte("ना")) || - bytes.HasSuffix(input, []byte("ते")) || - bytes.HasSuffix(input, []byte("ीं")) || - bytes.HasSuffix(input, []byte("ती")) || - bytes.HasSuffix(input, []byte("ता")) || - bytes.HasSuffix(input, []byte("ाँ")) || - bytes.HasSuffix(input, []byte("ां")) || - bytes.HasSuffix(input, []byte("ों")) || - bytes.HasSuffix(input, []byte("ें"))) { - return analysis.TruncateRunes(input, 2) - } - - // 1 - if inputLen > 2 && - (bytes.HasSuffix(input, []byte("ो")) || - bytes.HasSuffix(input, []byte("े")) || - bytes.HasSuffix(input, []byte("ू")) || - bytes.HasSuffix(input, []byte("ु")) || - bytes.HasSuffix(input, []byte("ी")) || - bytes.HasSuffix(input, []byte("ि")) || - bytes.HasSuffix(input, []byte("ा"))) { - return analysis.TruncateRunes(input, 1) - } - - return input -} - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewHindiStemmerFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter_test.go deleted file mode 100644 index 0040445a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/hindi_stemmer_filter_test.go +++ /dev/null @@ -1,303 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hi - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestHindiStemmerFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // masc noun inflections - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडका"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडक"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडके"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडक"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडकों"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडक"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("गुरु"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("गुर"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("गुरुओं"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("गुर"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("दोस्त"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("दोस्त"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("दोस्तों"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("दोस्त"), - }, - }, - }, - // feminine noun inflections - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडकी"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडक"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडकियों"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("लडक"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("किताब"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("किताब"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("किताबें"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("किताब"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("किताबों"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("किताब"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आध्यापीका"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आध्यापीक"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आध्यापीकाएं"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आध्यापीक"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आध्यापीकाओं"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आध्यापीक"), - }, - }, - }, - // some verb forms - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खाना"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खा"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खाता"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खा"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खाती"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खा"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खा"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("खा"), - }, - }, - }, - // exceptions - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("कठिनाइयां"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("कठिन"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("कठिन"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("कठिन"), - }, - }, - }, - // empty - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - hindiStemmerFilter := NewHindiStemmerFilter() - for _, test := range tests { - actual := hindiStemmerFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_filter_hi.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_filter_hi.go deleted file mode 100644 index 5dfd2684..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_filter_hi.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hi - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_words_hi.go b/vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_words_hi.go deleted file mode 100644 index f0300f0d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hi/stop_words_hi.go +++ /dev/null @@ -1,259 +0,0 @@ -package hi - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_hi" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var HindiStopWords = []byte(`# Also see http://www.opensource.org/licenses/bsd-license.html -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# This file was created by Jacques Savoy and is distributed under the BSD license. -# Note: by default this file also contains forms normalized by HindiNormalizer -# for spelling variation (see section below), such that it can be used whether or -# not you enable that feature. When adding additional entries to this list, -# please add the normalized form as well. -अंदर -अत -अपना -अपनी -अपने -अभी -आदि -आप -इत्यादि -इन -इनका -इन्हीं -इन्हें -इन्हों -इस -इसका -इसकी -इसके -इसमें -इसी -इसे -उन -उनका -उनकी -उनके -उनको -उन्हीं -उन्हें -उन्हों -उस -उसके -उसी -उसे -एक -एवं -एस -ऐसे -और -कई -कर -करता -करते -करना -करने -करें -कहते -कहा -का -काफ़ी -कि -कितना -किन्हें -किन्हों -किया -किर -किस -किसी -किसे -की -कुछ -कुल -के -को -कोई -कौन -कौनसा -गया -घर -जब -जहाँ -जा -जितना -जिन -जिन्हें -जिन्हों -जिस -जिसे -जीधर -जैसा -जैसे -जो -तक -तब -तरह -तिन -तिन्हें -तिन्हों -तिस -तिसे -तो -था -थी -थे -दबारा -दिया -दुसरा -दूसरे -दो -द्वारा -न -नहीं -ना -निहायत -नीचे -ने -पर -पर -पहले -पूरा -पे -फिर -बनी -बही -बहुत -बाद -बाला -बिलकुल -भी -भीतर -मगर -मानो -मे -में -यदि -यह -यहाँ -यही -या -यिह -ये -रखें -रहा -रहे -ऱ्वासा -लिए -लिये -लेकिन -व -वर्ग -वह -वह -वहाँ -वहीं -वाले -वुह -वे -वग़ैरह -संग -सकता -सकते -सबसे -सभी -साथ -साबुत -साभ -सारा -से -सो -ही -हुआ -हुई -हुए -है -हैं -हो -होता -होती -होते -होना -होने -# additional normalized forms of the above -अपनि -जेसे -होति -सभि -तिंहों -इंहों -दवारा -इसि -किंहें -थि -उंहों -ओर -जिंहें -वहिं -अभि -बनि -हि -उंहिं -उंहें -हें -वगेरह -एसे -रवासा -कोन -निचे -काफि -उसि -पुरा -भितर -हे -बहि -वहां -कोइ -यहां -जिंहों -तिंहें -किसि -कइ -यहि -इंहिं -जिधर -इंहें -अदि -इतयादि -हुइ -कोनसा -इसकि -दुसरे -जहां -अप -किंहों -उनकि -भि -वरग -हुअ -जेसा -नहिं -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(HindiStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu.go b/vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu.go deleted file mode 100644 index cc85f125..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package hu - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "hu" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopHuFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerHuFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopHuFilter, - stemmerHuFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu_test.go deleted file mode 100644 index 5395b371..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hu/analyzer_hu_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package hu - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestHungarianAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("babakocsi"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("babakocs"), - }, - }, - }, - { - input: []byte("babakocsijáért"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("babakocs"), - }, - }, - }, - // stop word - { - input: []byte("által"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hu/stemmer_hu.go b/vendor/github.com/blevesearch/bleve/analysis/language/hu/stemmer_hu.go deleted file mode 100644 index a73c5ec1..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hu/stemmer_hu.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package hu - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_hu" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("hu") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_filter_hu.go b/vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_filter_hu.go deleted file mode 100644 index 5e37a9a9..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_filter_hu.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hu - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_words_hu.go b/vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_words_hu.go deleted file mode 100644 index fe45d55e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hu/stop_words_hu.go +++ /dev/null @@ -1,235 +0,0 @@ -package hu - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_hu" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var HungarianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/hungarian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - -| Hungarian stop word list -| prepared by Anna Tordai - -a -ahogy -ahol -aki -akik -akkor -alatt -által -általában -amely -amelyek -amelyekben -amelyeket -amelyet -amelynek -ami -amit -amolyan -amíg -amikor -át -abban -ahhoz -annak -arra -arról -az -azok -azon -azt -azzal -azért -aztán -azután -azonban -bár -be -belül -benne -cikk -cikkek -cikkeket -csak -de -e -eddig -egész -egy -egyes -egyetlen -egyéb -egyik -egyre -ekkor -el -elég -ellen -elő -először -előtt -első -én -éppen -ebben -ehhez -emilyen -ennek -erre -ez -ezt -ezek -ezen -ezzel -ezért -és -fel -felé -hanem -hiszen -hogy -hogyan -igen -így -illetve -ill. -ill -ilyen -ilyenkor -ison -ismét -itt -jó -jól -jobban -kell -kellett -keresztül -keressünk -ki -kívül -között -közül -legalább -lehet -lehetett -legyen -lenne -lenni -lesz -lett -maga -magát -majd -majd -már -más -másik -meg -még -mellett -mert -mely -melyek -mi -mit -míg -miért -milyen -mikor -minden -mindent -mindenki -mindig -mint -mintha -mivel -most -nagy -nagyobb -nagyon -ne -néha -nekem -neki -nem -néhány -nélkül -nincs -olyan -ott -össze -ő -ők -őket -pedig -persze -rá -s -saját -sem -semmi -sok -sokat -sokkal -számára -szemben -szerint -szinte -talán -tehát -teljes -tovább -továbbá -több -úgy -ugyanis -új -újabb -újra -után -utána -utolsó -vagy -vagyis -valaki -valami -valamint -való -vagyok -van -vannak -volt -voltam -voltak -voltunk -vissza -vele -viszont -volna -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(HungarianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_filter_hy.go b/vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_filter_hy.go deleted file mode 100644 index 61db29e5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_filter_hy.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package hy - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_words_hy.go b/vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_words_hy.go deleted file mode 100644 index 6dc3db9b..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/hy/stop_words_hy.go +++ /dev/null @@ -1,70 +0,0 @@ -package hy - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_hy" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var ArmenianStopWords = []byte(`# example set of Armenian stopwords. -այդ -այլ -այն -այս -դու -դուք -եմ -են -ենք -ես -եք -է -էի -էին -էինք -էիր -էիք -էր -ըստ -թ -ի -ին -իսկ -իր -կամ -համար -հետ -հետո -մենք -մեջ -մի -ն -նա -նաև -նրա -նրանք -որ -որը -որոնք -որպես -ու -ում -պիտի -վրա -և -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(ArmenianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/id/stop_filter_id.go b/vendor/github.com/blevesearch/bleve/analysis/language/id/stop_filter_id.go deleted file mode 100644 index 169a26a7..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/id/stop_filter_id.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package id - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/id/stop_words_id.go b/vendor/github.com/blevesearch/bleve/analysis/language/id/stop_words_id.go deleted file mode 100644 index d802dcfc..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/id/stop_words_id.go +++ /dev/null @@ -1,383 +0,0 @@ -package id - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_id" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var IndonesianStopWords = []byte(`# from appendix D of: A Study of Stemming Effects on Information -# Retrieval in Bahasa Indonesia -ada -adanya -adalah -adapun -agak -agaknya -agar -akan -akankah -akhirnya -aku -akulah -amat -amatlah -anda -andalah -antar -diantaranya -antara -antaranya -diantara -apa -apaan -mengapa -apabila -apakah -apalagi -apatah -atau -ataukah -ataupun -bagai -bagaikan -sebagai -sebagainya -bagaimana -bagaimanapun -sebagaimana -bagaimanakah -bagi -bahkan -bahwa -bahwasanya -sebaliknya -banyak -sebanyak -beberapa -seberapa -begini -beginian -beginikah -beginilah -sebegini -begitu -begitukah -begitulah -begitupun -sebegitu -belum -belumlah -sebelum -sebelumnya -sebenarnya -berapa -berapakah -berapalah -berapapun -betulkah -sebetulnya -biasa -biasanya -bila -bilakah -bisa -bisakah -sebisanya -boleh -bolehkah -bolehlah -buat -bukan -bukankah -bukanlah -bukannya -cuma -percuma -dahulu -dalam -dan -dapat -dari -daripada -dekat -demi -demikian -demikianlah -sedemikian -dengan -depan -di -dia -dialah -dini -diri -dirinya -terdiri -dong -dulu -enggak -enggaknya -entah -entahlah -terhadap -terhadapnya -hal -hampir -hanya -hanyalah -harus -haruslah -harusnya -seharusnya -hendak -hendaklah -hendaknya -hingga -sehingga -ia -ialah -ibarat -ingin -inginkah -inginkan -ini -inikah -inilah -itu -itukah -itulah -jangan -jangankan -janganlah -jika -jikalau -juga -justru -kala -kalau -kalaulah -kalaupun -kalian -kami -kamilah -kamu -kamulah -kan -kapan -kapankah -kapanpun -dikarenakan -karena -karenanya -ke -kecil -kemudian -kenapa -kepada -kepadanya -ketika -seketika -khususnya -kini -kinilah -kiranya -sekiranya -kita -kitalah -kok -lagi -lagian -selagi -lah -lain -lainnya -melainkan -selaku -lalu -melalui -terlalu -lama -lamanya -selama -selama -selamanya -lebih -terlebih -bermacam -macam -semacam -maka -makanya -makin -malah -malahan -mampu -mampukah -mana -manakala -manalagi -masih -masihkah -semasih -masing -mau -maupun -semaunya -memang -mereka -merekalah -meski -meskipun -semula -mungkin -mungkinkah -nah -namun -nanti -nantinya -nyaris -oleh -olehnya -seorang -seseorang -pada -padanya -padahal -paling -sepanjang -pantas -sepantasnya -sepantasnyalah -para -pasti -pastilah -per -pernah -pula -pun -merupakan -rupanya -serupa -saat -saatnya -sesaat -saja -sajalah -saling -bersama -sama -sesama -sambil -sampai -sana -sangat -sangatlah -saya -sayalah -se -sebab -sebabnya -sebuah -tersebut -tersebutlah -sedang -sedangkan -sedikit -sedikitnya -segala -segalanya -segera -sesegera -sejak -sejenak -sekali -sekalian -sekalipun -sesekali -sekaligus -sekarang -sekarang -sekitar -sekitarnya -sela -selain -selalu -seluruh -seluruhnya -semakin -sementara -sempat -semua -semuanya -sendiri -sendirinya -seolah -seperti -sepertinya -sering -seringnya -serta -siapa -siapakah -siapapun -disini -disinilah -sini -sinilah -sesuatu -sesuatunya -suatu -sesudah -sesudahnya -sudah -sudahkah -sudahlah -supaya -tadi -tadinya -tak -tanpa -setelah -telah -tentang -tentu -tentulah -tentunya -tertentu -seterusnya -tapi -tetapi -setiap -tiap -setidaknya -tidak -tidakkah -tidaklah -toh -waduh -wah -wahai -sewaktu -walau -walaupun -wong -yaitu -yakni -yang -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(IndonesianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize.go b/vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize.go deleted file mode 100644 index 58acfce7..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package in - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const NormalizeName = "normalize_in" - -type IndicNormalizeFilter struct { -} - -func NewIndicNormalizeFilter() *IndicNormalizeFilter { - return &IndicNormalizeFilter{} -} - -func (s *IndicNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - runes := bytes.Runes(token.Term) - runes = normalize(runes) - token.Term = analysis.BuildTermFromRunes(runes) - } - return input -} - -func NormalizerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewIndicNormalizeFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(NormalizeName, NormalizerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize_test.go deleted file mode 100644 index 246b3d6b..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/in/indic_normalize_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package in - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestIndicNormalizeFilter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - // basics - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अाॅअाॅ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ऑऑ"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अाॆअाॆ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ऒऒ"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अाेअाे"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ओओ"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अाैअाै"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("औऔ"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अाअा"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("आआ"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("अाैर"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("और"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ত্‍"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ৎ"), - }, - }, - }, - // empty term - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte(""), - }, - }, - }, - } - - indicNormalizeFilter := NewIndicNormalizeFilter() - for _, test := range tests { - actual := indicNormalizeFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - t.Errorf("expected % x, got % x for % x", test.output[0].Term, actual[0].Term, test.input[0].Term) - t.Errorf("expected %s, got %s for %s", test.output[0].Term, actual[0].Term, test.input[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/in/scripts.go b/vendor/github.com/blevesearch/bleve/analysis/language/in/scripts.go deleted file mode 100644 index 1ac08782..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/in/scripts.go +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package in - -import ( - "unicode" - - "github.com/blevesearch/bleve/analysis" - "github.com/willf/bitset" -) - -type ScriptData struct { - flag rune - base rune - decompMask *bitset.BitSet -} - -var scripts = map[*unicode.RangeTable]*ScriptData{ - unicode.Devanagari: &ScriptData{ - flag: 1, - base: 0x0900, - }, - unicode.Bengali: &ScriptData{ - flag: 2, - base: 0x0980, - }, - unicode.Gurmukhi: &ScriptData{ - flag: 4, - base: 0x0A00, - }, - unicode.Gujarati: &ScriptData{ - flag: 8, - base: 0x0A80, - }, - unicode.Oriya: &ScriptData{ - flag: 16, - base: 0x0B00, - }, - unicode.Tamil: &ScriptData{ - flag: 32, - base: 0x0B80, - }, - unicode.Telugu: &ScriptData{ - flag: 64, - base: 0x0C00, - }, - unicode.Kannada: &ScriptData{ - flag: 128, - base: 0x0C80, - }, - unicode.Malayalam: &ScriptData{ - flag: 256, - base: 0x0D00, - }, -} - -func flag(ub *unicode.RangeTable) rune { - return scripts[ub].flag -} - -var decompositions = [][]rune{ - /* devanagari, gujarati vowel candra O */ - {0x05, 0x3E, 0x45, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari short O */ - {0x05, 0x3E, 0x46, 0x12, flag(unicode.Devanagari)}, - /* devanagari, gujarati letter O */ - {0x05, 0x3E, 0x47, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari letter AI, gujarati letter AU */ - {0x05, 0x3E, 0x48, 0x14, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari, bengali, gurmukhi, gujarati, oriya AA */ - {0x05, 0x3E, -1, 0x06, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Gurmukhi) | flag(unicode.Gujarati) | flag(unicode.Oriya)}, - /* devanagari letter candra A */ - {0x05, 0x45, -1, 0x72, flag(unicode.Devanagari)}, - /* gujarati vowel candra E */ - {0x05, 0x45, -1, 0x0D, flag(unicode.Gujarati)}, - /* devanagari letter short A */ - {0x05, 0x46, -1, 0x04, flag(unicode.Devanagari)}, - /* gujarati letter E */ - {0x05, 0x47, -1, 0x0F, flag(unicode.Gujarati)}, - /* gurmukhi, gujarati letter AI */ - {0x05, 0x48, -1, 0x10, flag(unicode.Gurmukhi) | flag(unicode.Gujarati)}, - /* devanagari, gujarati vowel candra O */ - {0x05, 0x49, -1, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari short O */ - {0x05, 0x4A, -1, 0x12, flag(unicode.Devanagari)}, - /* devanagari, gujarati letter O */ - {0x05, 0x4B, -1, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari letter AI, gurmukhi letter AU, gujarati letter AU */ - {0x05, 0x4C, -1, 0x14, flag(unicode.Devanagari) | flag(unicode.Gurmukhi) | flag(unicode.Gujarati)}, - /* devanagari, gujarati vowel candra O */ - {0x06, 0x45, -1, 0x11, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari short O */ - {0x06, 0x46, -1, 0x12, flag(unicode.Devanagari)}, - /* devanagari, gujarati letter O */ - {0x06, 0x47, -1, 0x13, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari letter AI, gujarati letter AU */ - {0x06, 0x48, -1, 0x14, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* malayalam letter II */ - {0x07, 0x57, -1, 0x08, flag(unicode.Malayalam)}, - /* devanagari letter UU */ - {0x09, 0x41, -1, 0x0A, flag(unicode.Devanagari)}, - /* tamil, malayalam letter UU (some styles) */ - {0x09, 0x57, -1, 0x0A, flag(unicode.Tamil) | flag(unicode.Malayalam)}, - /* malayalam letter AI */ - {0x0E, 0x46, -1, 0x10, flag(unicode.Malayalam)}, - /* devanagari candra E */ - {0x0F, 0x45, -1, 0x0D, flag(unicode.Devanagari)}, - /* devanagari short E */ - {0x0F, 0x46, -1, 0x0E, flag(unicode.Devanagari)}, - /* devanagari AI */ - {0x0F, 0x47, -1, 0x10, flag(unicode.Devanagari)}, - /* oriya AI */ - {0x0F, 0x57, -1, 0x10, flag(unicode.Oriya)}, - /* malayalam letter OO */ - {0x12, 0x3E, -1, 0x13, flag(unicode.Malayalam)}, - /* telugu, kannada letter AU */ - {0x12, 0x4C, -1, 0x14, flag(unicode.Telugu) | flag(unicode.Kannada)}, - /* telugu letter OO */ - {0x12, 0x55, -1, 0x13, flag(unicode.Telugu)}, - /* tamil, malayalam letter AU */ - {0x12, 0x57, -1, 0x14, flag(unicode.Tamil) | flag(unicode.Malayalam)}, - /* oriya letter AU */ - {0x13, 0x57, -1, 0x14, flag(unicode.Oriya)}, - /* devanagari qa */ - {0x15, 0x3C, -1, 0x58, flag(unicode.Devanagari)}, - /* devanagari, gurmukhi khha */ - {0x16, 0x3C, -1, 0x59, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, - /* devanagari, gurmukhi ghha */ - {0x17, 0x3C, -1, 0x5A, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, - /* devanagari, gurmukhi za */ - {0x1C, 0x3C, -1, 0x5B, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, - /* devanagari dddha, bengali, oriya rra */ - {0x21, 0x3C, -1, 0x5C, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Oriya)}, - /* devanagari, bengali, oriya rha */ - {0x22, 0x3C, -1, 0x5D, flag(unicode.Devanagari) | flag(unicode.Bengali) | flag(unicode.Oriya)}, - /* malayalam chillu nn */ - {0x23, 0x4D, 0xFF, 0x7A, flag(unicode.Malayalam)}, - /* bengali khanda ta */ - {0x24, 0x4D, 0xFF, 0x4E, flag(unicode.Bengali)}, - /* devanagari nnna */ - {0x28, 0x3C, -1, 0x29, flag(unicode.Devanagari)}, - /* malayalam chillu n */ - {0x28, 0x4D, 0xFF, 0x7B, flag(unicode.Malayalam)}, - /* devanagari, gurmukhi fa */ - {0x2B, 0x3C, -1, 0x5E, flag(unicode.Devanagari) | flag(unicode.Gurmukhi)}, - /* devanagari, bengali yya */ - {0x2F, 0x3C, -1, 0x5F, flag(unicode.Devanagari) | flag(unicode.Bengali)}, - /* telugu letter vocalic R */ - {0x2C, 0x41, 0x41, 0x0B, flag(unicode.Telugu)}, - /* devanagari rra */ - {0x30, 0x3C, -1, 0x31, flag(unicode.Devanagari)}, - /* malayalam chillu rr */ - {0x30, 0x4D, 0xFF, 0x7C, flag(unicode.Malayalam)}, - /* malayalam chillu l */ - {0x32, 0x4D, 0xFF, 0x7D, flag(unicode.Malayalam)}, - /* devanagari llla */ - {0x33, 0x3C, -1, 0x34, flag(unicode.Devanagari)}, - /* malayalam chillu ll */ - {0x33, 0x4D, 0xFF, 0x7E, flag(unicode.Malayalam)}, - /* telugu letter MA */ - {0x35, 0x41, -1, 0x2E, flag(unicode.Telugu)}, - /* devanagari, gujarati vowel sign candra O */ - {0x3E, 0x45, -1, 0x49, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari vowel sign short O */ - {0x3E, 0x46, -1, 0x4A, flag(unicode.Devanagari)}, - /* devanagari, gujarati vowel sign O */ - {0x3E, 0x47, -1, 0x4B, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* devanagari, gujarati vowel sign AU */ - {0x3E, 0x48, -1, 0x4C, flag(unicode.Devanagari) | flag(unicode.Gujarati)}, - /* kannada vowel sign II */ - {0x3F, 0x55, -1, 0x40, flag(unicode.Kannada)}, - /* gurmukhi vowel sign UU (when stacking) */ - {0x41, 0x41, -1, 0x42, flag(unicode.Gurmukhi)}, - /* tamil, malayalam vowel sign O */ - {0x46, 0x3E, -1, 0x4A, flag(unicode.Tamil) | flag(unicode.Malayalam)}, - /* kannada vowel sign OO */ - {0x46, 0x42, 0x55, 0x4B, flag(unicode.Kannada)}, - /* kannada vowel sign O */ - {0x46, 0x42, -1, 0x4A, flag(unicode.Kannada)}, - /* malayalam vowel sign AI (if reordered twice) */ - {0x46, 0x46, -1, 0x48, flag(unicode.Malayalam)}, - /* telugu, kannada vowel sign EE */ - {0x46, 0x55, -1, 0x47, flag(unicode.Telugu) | flag(unicode.Kannada)}, - /* telugu, kannada vowel sign AI */ - {0x46, 0x56, -1, 0x48, flag(unicode.Telugu) | flag(unicode.Kannada)}, - /* tamil, malayalam vowel sign AU */ - {0x46, 0x57, -1, 0x4C, flag(unicode.Tamil) | flag(unicode.Malayalam)}, - /* bengali, oriya vowel sign O, tamil, malayalam vowel sign OO */ - {0x47, 0x3E, -1, 0x4B, flag(unicode.Bengali) | flag(unicode.Oriya) | flag(unicode.Tamil) | flag(unicode.Malayalam)}, - /* bengali, oriya vowel sign AU */ - {0x47, 0x57, -1, 0x4C, flag(unicode.Bengali) | flag(unicode.Oriya)}, - /* kannada vowel sign OO */ - {0x4A, 0x55, -1, 0x4B, flag(unicode.Kannada)}, - /* gurmukhi letter I */ - {0x72, 0x3F, -1, 0x07, flag(unicode.Gurmukhi)}, - /* gurmukhi letter II */ - {0x72, 0x40, -1, 0x08, flag(unicode.Gurmukhi)}, - /* gurmukhi letter EE */ - {0x72, 0x47, -1, 0x0F, flag(unicode.Gurmukhi)}, - /* gurmukhi letter U */ - {0x73, 0x41, -1, 0x09, flag(unicode.Gurmukhi)}, - /* gurmukhi letter UU */ - {0x73, 0x42, -1, 0x0A, flag(unicode.Gurmukhi)}, - /* gurmukhi letter OO */ - {0x73, 0x4B, -1, 0x13, flag(unicode.Gurmukhi)}, -} - -func init() { - for _, scriptData := range scripts { - scriptData.decompMask = bitset.New(0x7d) - for _, decomposition := range decompositions { - ch := decomposition[0] - flags := decomposition[4] - if (flags & scriptData.flag) != 0 { - scriptData.decompMask.Set(uint(ch)) - } - } - } -} - -func lookupScript(r rune) *unicode.RangeTable { - for script, _ := range scripts { - if unicode.Is(script, r) { - return script - } - } - return nil -} - -func normalize(input []rune) []rune { - inputLen := len(input) - for i := 0; i < inputLen; i++ { - r := input[i] - script := lookupScript(r) - if script != nil { - scriptData := scripts[script] - ch := r - scriptData.base - if scriptData.decompMask.Test(uint(ch)) { - input = compose(ch, script, scriptData, input, i, inputLen) - inputLen = len(input) - } - } - } - return input[0:inputLen] -} - -func compose(ch0 rune, script0 *unicode.RangeTable, scriptData *ScriptData, input []rune, pos int, inputLen int) []rune { - if pos+1 >= inputLen { - return input // need at least 2 characters - } - - ch1 := input[pos+1] - scriptData.base - script1 := lookupScript(input[pos+1]) - if script0 != script1 { - return input // need to be same script - } - - ch2 := rune(-1) - if pos+2 < inputLen { - ch2 = input[pos+2] - scriptData.base - script2 := lookupScript(input[pos+2]) - if input[pos+2] == '\u200D' { - ch2 = 0xff // zero width joiner - } else if script2 != script1 { - ch2 = -1 // still allow 2 character match - } - } - - for _, decomposition := range decompositions { - if decomposition[0] == ch0 && - (decomposition[4]&scriptData.flag) != 0 { - if decomposition[1] == ch1 && - (decomposition[2] < 0 || decomposition[2] == ch2) { - input[pos] = scriptData.base + decomposition[3] - input = analysis.DeleteRune(input, pos+1) - if decomposition[2] >= 0 { - input = analysis.DeleteRune(input, pos+1) - } - return input - } - } - } - return input -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it.go deleted file mode 100644 index 9cb4021d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package it - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" -) - -const AnalyzerName = "it" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - elisionFilter, err := cache.TokenFilterNamed(ElisionName) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopItFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerItFilter, err := cache.TokenFilterNamed(LightStemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - elisionFilter, - toLowerFilter, - stopItFilter, - stemmerItFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it_test.go deleted file mode 100644 index 03cc8112..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/analyzer_it_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package it - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestItalianAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("abbandonata"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abbandonat"), - }, - }, - }, - { - input: []byte("abbandonati"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abbandonat"), - }, - }, - }, - // stop word - { - input: []byte("dallo"), - output: analysis.TokenStream{}, - }, - // contractions - { - input: []byte("dell'Italia"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ital"), - }, - }, - }, - { - input: []byte("l'Italiano"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("italian"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/articles_it.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/articles_it.go deleted file mode 100644 index bfd4bb48..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/articles_it.go +++ /dev/null @@ -1,45 +0,0 @@ -package it - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const ArticlesName = "articles_it" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis - -var ItalianArticles = []byte(` -c -l -all -dall -dell -nell -sull -coll -pell -gl -agl -dagl -degl -negl -sugl -un -m -t -s -v -d -`) - -func ArticlesTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(ItalianArticles) - return rv, err -} - -func init() { - registry.RegisterTokenMap(ArticlesName, ArticlesTokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it.go deleted file mode 100644 index cbf8ab8c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package it - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/elision_filter" - "github.com/blevesearch/bleve/registry" -) - -const ElisionName = "elision_it" - -func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - articlesTokenMap, err := cache.TokenMapNamed(ArticlesName) - if err != nil { - return nil, fmt.Errorf("error building elision filter: %v", err) - } - return elision_filter.NewElisionFilter(articlesTokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(ElisionName, ElisionFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it_test.go deleted file mode 100644 index 5f261ba2..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/elision_it_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package it - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestItalianElision(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("dell'Italia"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Italia"), - }, - }, - }, - } - - cache := registry.NewCache() - elisionFilter, err := cache.TokenFilterNamed(ElisionName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := elisionFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it.go deleted file mode 100644 index 282dda18..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package it - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const LightStemmerName = "stemmer_it_light" - -type ItalianLightStemmerFilter struct { -} - -func NewItalianLightStemmerFilterFilter() *ItalianLightStemmerFilter { - return &ItalianLightStemmerFilter{} -} - -func (s *ItalianLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - runes := bytes.Runes(token.Term) - runes = stem(runes) - token.Term = analysis.BuildTermFromRunes(runes) - } - return input -} - -func stem(input []rune) []rune { - - inputLen := len(input) - - if inputLen < 6 { - return input - } - - for i := 0; i < inputLen; i++ { - switch input[i] { - case 'à', 'á', 'â', 'ä': - input[i] = 'a' - case 'ò', 'ó', 'ô', 'ö': - input[i] = 'o' - case 'è', 'é', 'ê', 'ë': - input[i] = 'e' - case 'ù', 'ú', 'û', 'ü': - input[i] = 'u' - case 'ì', 'í', 'î', 'ï': - input[i] = 'i' - } - } - - switch input[inputLen-1] { - case 'e': - if input[inputLen-2] == 'i' || input[inputLen-2] == 'h' { - return input[0 : inputLen-2] - } else { - return input[0 : inputLen-1] - } - case 'i': - if input[inputLen-2] == 'h' || input[inputLen-2] == 'i' { - return input[0 : inputLen-2] - } else { - return input[0 : inputLen-1] - } - case 'a': - if input[inputLen-2] == 'i' { - return input[0 : inputLen-2] - } else { - return input[0 : inputLen-1] - } - case 'o': - if input[inputLen-2] == 'i' { - return input[0 : inputLen-2] - } else { - return input[0 : inputLen-1] - } - } - - return input -} - -func ItalianLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewItalianLightStemmerFilterFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(LightStemmerName, ItalianLightStemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it_test.go deleted file mode 100644 index 1f81370c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/light_stemmer_it_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package it - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestItalianLightStemmer(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ragazzo"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ragazz"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ragazzi"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ragazz"), - }, - }, - }, - } - - cache := registry.NewCache() - filter, err := cache.TokenFilterNamed(LightStemmerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := filter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/stemmer_it.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/stemmer_it.go deleted file mode 100644 index d979f388..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/stemmer_it.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package it - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_it" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("it") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/stop_filter_it.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/stop_filter_it.go deleted file mode 100644 index 80cb042a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/stop_filter_it.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package it - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/it/stop_words_it.go b/vendor/github.com/blevesearch/bleve/analysis/language/it/stop_words_it.go deleted file mode 100644 index 5cca4bab..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/it/stop_words_it.go +++ /dev/null @@ -1,327 +0,0 @@ -package it - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_it" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var ItalianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | An Italian stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - -ad | a (to) before vowel -al | a + il -allo | a + lo -ai | a + i -agli | a + gli -all | a + l' -agl | a + gl' -alla | a + la -alle | a + le -con | with -col | con + il -coi | con + i (forms collo, cogli etc are now very rare) -da | from -dal | da + il -dallo | da + lo -dai | da + i -dagli | da + gli -dall | da + l' -dagl | da + gll' -dalla | da + la -dalle | da + le -di | of -del | di + il -dello | di + lo -dei | di + i -degli | di + gli -dell | di + l' -degl | di + gl' -della | di + la -delle | di + le -in | in -nel | in + el -nello | in + lo -nei | in + i -negli | in + gli -nell | in + l' -negl | in + gl' -nella | in + la -nelle | in + le -su | on -sul | su + il -sullo | su + lo -sui | su + i -sugli | su + gli -sull | su + l' -sugl | su + gl' -sulla | su + la -sulle | su + le -per | through, by -tra | among -contro | against -io | I -tu | thou -lui | he -lei | she -noi | we -voi | you -loro | they -mio | my -mia | -miei | -mie | -tuo | -tua | -tuoi | thy -tue | -suo | -sua | -suoi | his, her -sue | -nostro | our -nostra | -nostri | -nostre | -vostro | your -vostra | -vostri | -vostre | -mi | me -ti | thee -ci | us, there -vi | you, there -lo | him, the -la | her, the -li | them -le | them, the -gli | to him, the -ne | from there etc -il | the -un | a -uno | a -una | a -ma | but -ed | and -se | if -perché | why, because -anche | also -come | how -dov | where (as dov') -dove | where -che | who, that -chi | who -cui | whom -non | not -più | more -quale | who, that -quanto | how much -quanti | -quanta | -quante | -quello | that -quelli | -quella | -quelle | -questo | this -questi | -questa | -queste | -si | yes -tutto | all -tutti | all - - | single letter forms: - -a | at -c | as c' for ce or ci -e | and -i | the -l | as l' -o | or - - | forms of avere, to have (not including the infinitive): - -ho -hai -ha -abbiamo -avete -hanno -abbia -abbiate -abbiano -avrò -avrai -avrà -avremo -avrete -avranno -avrei -avresti -avrebbe -avremmo -avreste -avrebbero -avevo -avevi -aveva -avevamo -avevate -avevano -ebbi -avesti -ebbe -avemmo -aveste -ebbero -avessi -avesse -avessimo -avessero -avendo -avuto -avuta -avuti -avute - - | forms of essere, to be (not including the infinitive): -sono -sei -è -siamo -siete -sia -siate -siano -sarò -sarai -sarà -saremo -sarete -saranno -sarei -saresti -sarebbe -saremmo -sareste -sarebbero -ero -eri -era -eravamo -eravate -erano -fui -fosti -fu -fummo -foste -furono -fossi -fosse -fossimo -fossero -essendo - - | forms of fare, to do (not including the infinitive, fa, fat-): -faccio -fai -facciamo -fanno -faccia -facciate -facciano -farò -farai -farà -faremo -farete -faranno -farei -faresti -farebbe -faremmo -fareste -farebbero -facevo -facevi -faceva -facevamo -facevate -facevano -feci -facesti -fece -facemmo -faceste -fecero -facessi -facesse -facessimo -facessero -facendo - - | forms of stare, to be (not including the infinitive): -sto -stai -sta -stiamo -stanno -stia -stiate -stiano -starò -starai -starà -staremo -starete -staranno -starei -staresti -starebbe -staremmo -stareste -starebbero -stavo -stavi -stava -stavamo -stavate -stavano -stetti -stesti -stette -stemmo -steste -stettero -stessi -stesse -stessimo -stessero -stando -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(ItalianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja.go b/vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja.go deleted file mode 100644 index 2592ba0d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build kagome full - -package ja - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize" - "github.com/blevesearch/bleve/registry" -) - -const AnalyzerName = "ja" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - kagomeTokenizer, err := cache.TokenizerNamed(TokenizerName) - if err != nil { - return nil, err - } - normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKD) - rv := analysis.Analyzer{ - Tokenizer: kagomeTokenizer, - TokenFilters: []analysis.TokenFilter{ - normalizeFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja_test.go deleted file mode 100644 index d88d8a13..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build kagome full - -package ja - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestJaAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - input: []byte("こんにちは世界"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こんにちは"), - Type: analysis.Ideographic, - Position: 1, - Start: 0, - End: 15, - }, - &analysis.Token{ - Term: []byte("世界"), - Type: analysis.Ideographic, - Position: 2, - Start: 15, - End: 21, - }, - }, - }, - { - input: []byte("ｶﾀｶﾅ"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("カタカナ"), - Type: analysis.Ideographic, - Position: 1, - Start: 0, - End: 12, - }, - }, - }, - } - - cache := registry.NewCache() - for _, test := range tests { - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome.go b/vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome.go deleted file mode 100644 index 1eec87da..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build kagome full - -package ja - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/ikawaha/kagome" -) - -const TokenizerName = "kagome" - -type KagomeMorphTokenizer struct { - tok *kagome.Tokenizer -} - -func NewKagomeMorphTokenizer() *KagomeMorphTokenizer { - return &KagomeMorphTokenizer{ - tok: kagome.NewTokenizer(), - } -} - -func NewKagomeMorphTokenizerWithUserDic(userdic *kagome.UserDic) *KagomeMorphTokenizer { - k := kagome.NewTokenizer() - k.SetUserDic(userdic) - return &KagomeMorphTokenizer{ - tok: k, - } -} - -func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream { - var ( - morphs []kagome.Token - prevstart int - ) - - rv := make(analysis.TokenStream, 0, len(input)) - if len(input) < 1 { - return rv - } - - morphs = t.tok.Tokenize(string(input)) - - for i, m := range morphs { - if m.Surface == "EOS" || m.Surface == "BOS" { - continue - } - - surfacelen := len(m.Surface) - token := &analysis.Token{ - Term: []byte(m.Surface), - Position: i, - Start: prevstart, - End: prevstart + surfacelen, - Type: analysis.Ideographic, - } - - prevstart = prevstart + surfacelen - rv = append(rv, token) - } - - return rv -} - -func KagomeMorphTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - return NewKagomeMorphTokenizer(), nil -} - -func init() { - registry.RegisterTokenizer(TokenizerName, KagomeMorphTokenizerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome_test.go deleted file mode 100644 index be8d4959..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build kagome full - -package ja - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestKagome(t *testing.T) { - - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - []byte("こんにちは世界"), - analysis.TokenStream{ - { - Start: 0, - End: 15, - Term: []byte("こんにちは"), - Position: 1, - Type: analysis.Ideographic, - }, - { - Start: 15, - End: 21, - Term: []byte("世界"), - Position: 2, - Type: analysis.Ideographic, - }, - }, - }, - } - - tokenizer := NewKagomeMorphTokenizer() - for _, test := range tests { - actuals := tokenizer.Tokenize(test.input) - - if !reflect.DeepEqual(actuals, test.output) { - t.Errorf("Expected %v, got %v for %s", test.output, actuals, string(test.input)) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl.go b/vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl.go deleted file mode 100644 index 009ea3d8..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package nl - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "nl" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopNlFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerNlFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopNlFilter, - stemmerNlFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl_test.go deleted file mode 100644 index f0eb7eb0..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/nl/analyzer_nl_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package nl - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestDutchAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("lichamelijk"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("licham"), - }, - }, - }, - { - input: []byte("lichamelijke"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("licham"), - }, - }, - }, - // stop word - { - input: []byte("van"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/nl/stemmer_nl.go b/vendor/github.com/blevesearch/bleve/analysis/language/nl/stemmer_nl.go deleted file mode 100644 index e6613f55..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/nl/stemmer_nl.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package nl - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_nl" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("nl") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_filter_nl.go b/vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_filter_nl.go deleted file mode 100644 index 8be87313..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_filter_nl.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package nl - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_words_nl.go b/vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_words_nl.go deleted file mode 100644 index 4adae100..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/nl/stop_words_nl.go +++ /dev/null @@ -1,143 +0,0 @@ -package nl - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_nl" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var DutchStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Dutch stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large sample of Dutch text. - - | Dutch stop words frequently exhibit homonym clashes. These are indicated - | clearly below. - -de | the -en | and -van | of, from -ik | I, the ego -te | (1) chez, at etc, (2) to, (3) too -dat | that, which -die | that, those, who, which -in | in, inside -een | a, an, one -hij | he -het | the, it -niet | not, nothing, naught -zijn | (1) to be, being, (2) his, one's, its -is | is -was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river -op | on, upon, at, in, up, used up -aan | on, upon, to (as dative) -met | with, by -als | like, such as, when -voor | (1) before, in front of, (2) furrow -had | had, past tense all persons sing. of 'hebben' (have) -er | there -maar | but, only -om | round, about, for etc -hem | him -dan | then -zou | should/would, past tense all persons sing. of 'zullen' -of | or, whether, if -wat | what, something, anything -mijn | possessive and noun 'mine' -men | people, 'one' -dit | this -zo | so, thus, in this way -door | through by -over | over, across -ze | she, her, they, them -zich | oneself -bij | (1) a bee, (2) by, near, at -ook | also, too -tot | till, until -je | you -mij | me -uit | out of, from -der | Old Dutch form of 'van der' still found in surnames -daar | (1) there, (2) because -haar | (1) her, their, them, (2) hair -naar | (1) unpleasant, unwell etc, (2) towards, (3) as -heb | present first person sing. of 'to have' -hoe | how, why -heeft | present third person sing. of 'to have' -hebben | 'to have' and various parts thereof -deze | this -u | you -want | (1) for, (2) mitten, (3) rigging -nog | yet, still -zal | 'shall', first and third person sing. of verb 'zullen' (will) -me | me -zij | she, they -nu | now -ge | 'thou', still used in Belgium and south Netherlands -geen | none -omdat | because -iets | something, somewhat -worden | to become, grow, get -toch | yet, still -al | all, every, each -waren | (1) 'were' (2) to wander, (3) wares, (3) -veel | much, many -meer | (1) more, (2) lake -doen | to do, to make -toen | then, when -moet | noun 'spot/mote' and present form of 'to must' -ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' -zonder | without -kan | noun 'can' and present form of 'to be able' -hun | their, them -dus | so, consequently -alles | all, everything, anything -onder | under, beneath -ja | yes, of course -eens | once, one day -hier | here -wie | who -werd | imperfect third person sing. of 'become' -altijd | always -doch | yet, but etc -wordt | present third person sing. of 'become' -wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans -kunnen | to be able -ons | us/our -zelf | self -tegen | against, towards, at -na | after, near -reeds | already -wil | (1) present tense of 'want', (2) 'will', noun, (3) fender -kon | could; past tense of 'to be able' -niets | nothing -uw | your -iemand | somebody -geweest | been; past participle of 'be' -andere | other -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(DutchStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no.go b/vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no.go deleted file mode 100644 index aa62927f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package no - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "no" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopNoFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerNoFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopNoFilter, - stemmerNoFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no_test.go deleted file mode 100644 index c3026f06..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/no/analyzer_no_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package no - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestNorwegianAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("havnedistriktene"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("havnedistrikt"), - }, - }, - }, - { - input: []byte("havnedistrikter"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("havnedistrikt"), - }, - }, - }, - // stop word - { - input: []byte("det"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/no/stemmer_no.go b/vendor/github.com/blevesearch/bleve/analysis/language/no/stemmer_no.go deleted file mode 100644 index e2b60683..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/no/stemmer_no.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package no - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_no" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("no") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/no/stop_filter_no.go b/vendor/github.com/blevesearch/bleve/analysis/language/no/stop_filter_no.go deleted file mode 100644 index f07b91ba..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/no/stop_filter_no.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package no - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/no/stop_words_no.go b/vendor/github.com/blevesearch/bleve/analysis/language/no/stop_words_no.go deleted file mode 100644 index bfca3484..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/no/stop_words_no.go +++ /dev/null @@ -1,218 +0,0 @@ -package no - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_no" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var NorwegianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/norwegian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Norwegian stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This stop word list is for the dominant bokmål dialect. Words unique - | to nynorsk are marked *. - - | Revised by Jan Bruusgaard , Jan 2005 - -og | and -i | in -jeg | I -det | it/this/that -at | to (w. inf.) -en | a/an -et | a/an -den | it/this/that -til | to -er | is/am/are -som | who/that -på | on -de | they / you(formal) -med | with -han | he -av | of -ikke | not -ikkje | not * -der | there -så | so -var | was/were -meg | me -seg | you -men | but -ett | one -har | have -om | about -vi | we -min | my -mitt | my -ha | have -hadde | had -hun | she -nå | now -over | over -da | when/as -ved | by/know -fra | from -du | you -ut | out -sin | your -dem | them -oss | us -opp | up -man | you/one -kan | can -hans | his -hvor | where -eller | or -hva | what -skal | shall/must -selv | self (reflective) -sjøl | self (reflective) -her | here -alle | all -vil | will -bli | become -ble | became -blei | became * -blitt | have become -kunne | could -inn | in -når | when -være | be -kom | come -noen | some -noe | some -ville | would -dere | you -som | who/which/that -deres | their/theirs -kun | only/just -ja | yes -etter | after -ned | down -skulle | should -denne | this -for | for/because -deg | you -si | hers/his -sine | hers/his -sitt | hers/his -mot | against -å | to -meget | much -hvorfor | why -dette | this -disse | these/those -uten | without -hvordan | how -ingen | none -din | your -ditt | your -blir | become -samme | same -hvilken | which -hvilke | which (plural) -sånn | such a -inni | inside/within -mellom | between -vår | our -hver | each -hvem | who -vors | us/ours -hvis | whose -både | both -bare | only/just -enn | than -fordi | as/because -før | before -mange | many -også | also -slik | just -vært | been -være | to be -båe | both * -begge | both -siden | since -dykk | your * -dykkar | yours * -dei | they * -deira | them * -deires | theirs * -deim | them * -di | your (fem.) * -då | as/when * -eg | I * -ein | a/an * -eit | a/an * -eitt | a/an * -elles | or * -honom | he * -hjå | at * -ho | she * -hoe | she * -henne | her -hennar | her/hers -hennes | hers -hoss | how * -hossen | how * -ikkje | not * -ingi | noone * -inkje | noone * -korleis | how * -korso | how * -kva | what/which * -kvar | where * -kvarhelst | where * -kven | who/whom * -kvi | why * -kvifor | why * -me | we * -medan | while * -mi | my * -mine | my * -mykje | much * -no | now * -nokon | some (masc./neut.) * -noka | some (fem.) * -nokor | some * -noko | some * -nokre | some * -si | his/hers * -sia | since * -sidan | since * -so | so * -somt | some * -somme | some * -um | about* -upp | up * -vere | be * -vore | was * -verte | become * -vort | become * -varte | became * -vart | became * - -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(NorwegianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/porter/stemmer_porter.go b/vendor/github.com/blevesearch/bleve/analysis/language/porter/stemmer_porter.go deleted file mode 100644 index 3e9dca42..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/porter/stemmer_porter.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package porter - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_porter_classic" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("porter") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt.go b/vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt.go deleted file mode 100644 index bd81427b..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package pt - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/unicode" -) - -const AnalyzerName = "pt" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - tokenizer, err := cache.TokenizerNamed(unicode.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopPtFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerPtFilter, err := cache.TokenFilterNamed(LightStemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: tokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopPtFilter, - stemmerPtFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt_test.go deleted file mode 100644 index d6a81549..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/pt/analyzer_pt_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package pt - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestPortugueseAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("quilométricas"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("quilometric"), - }, - }, - }, - { - input: []byte("quilométricos"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("quilometric"), - }, - }, - }, - // stop word - { - input: []byte("não"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt.go b/vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt.go deleted file mode 100644 index 93aac7f0..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package pt - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const LightStemmerName = "stemmer_pt_light" - -type PortugueseLightStemmerFilter struct { -} - -func NewPortugueseLightStemmerFilter() *PortugueseLightStemmerFilter { - return &PortugueseLightStemmerFilter{} -} - -func (s *PortugueseLightStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - runes := bytes.Runes(token.Term) - runes = stem(runes) - token.Term = analysis.BuildTermFromRunes(runes) - } - return input -} - -func stem(input []rune) []rune { - - inputLen := len(input) - - if inputLen < 4 { - return input - } - - input = removeSuffix(input) - inputLen = len(input) - - if inputLen > 3 && input[inputLen-1] == 'a' { - input = normFeminine(input) - inputLen = len(input) - } - - if inputLen > 4 { - switch input[inputLen-1] { - case 'e', 'a', 'o': - input = input[0 : inputLen-1] - inputLen = len(input) - } - } - - for i := 0; i < inputLen; i++ { - switch input[i] { - case 'à', 'á', 'â', 'ä', 'ã': - input[i] = 'a' - case 'ò', 'ó', 'ô', 'ö', 'õ': - input[i] = 'o' - case 'è', 'é', 'ê', 'ë': - input[i] = 'e' - case 'ù', 'ú', 'û', 'ü': - input[i] = 'u' - case 'ì', 'í', 'î', 'ï': - input[i] = 'i' - case 'ç': - input[i] = 'c' - } - } - - return input -} - -func removeSuffix(input []rune) []rune { - - inputLen := len(input) - - if inputLen > 4 && analysis.RunesEndsWith(input, "es") { - switch input[inputLen-3] { - case 'r', 's', 'l', 'z': - return input[0 : inputLen-2] - } - } - - if inputLen > 3 && analysis.RunesEndsWith(input, "ns") { - input[inputLen-2] = 'm' - return input[0 : inputLen-1] - } - - if inputLen > 4 && (analysis.RunesEndsWith(input, "eis") || analysis.RunesEndsWith(input, "éis")) { - input[inputLen-3] = 'e' - input[inputLen-2] = 'l' - return input[0 : inputLen-1] - } - - if inputLen > 4 && analysis.RunesEndsWith(input, "ais") { - input[inputLen-2] = 'l' - return input[0 : inputLen-1] - } - - if inputLen > 4 && analysis.RunesEndsWith(input, "óis") { - input[inputLen-3] = 'o' - input[inputLen-2] = 'l' - return input[0 : inputLen-1] - } - - if inputLen > 4 && analysis.RunesEndsWith(input, "is") { - input[inputLen-1] = 'l' - return input - } - - if inputLen > 3 && - (analysis.RunesEndsWith(input, "ões") || - analysis.RunesEndsWith(input, "ães")) { - input = input[0 : inputLen-1] - inputLen = len(input) - input[inputLen-2] = 'ã' - input[inputLen-1] = 'o' - return input - } - - if inputLen > 6 && analysis.RunesEndsWith(input, "mente") { - return input[0 : inputLen-5] - } - - if inputLen > 3 && input[inputLen-1] == 's' { - return input[0 : inputLen-1] - } - return input -} - -func normFeminine(input []rune) []rune { - inputLen := len(input) - - if inputLen > 7 && - (analysis.RunesEndsWith(input, "inha") || - analysis.RunesEndsWith(input, "iaca") || - analysis.RunesEndsWith(input, "eira")) { - input[inputLen-1] = 'o' - return input - } - - if inputLen > 6 { - if analysis.RunesEndsWith(input, "osa") || - analysis.RunesEndsWith(input, "ica") || - analysis.RunesEndsWith(input, "ida") || - analysis.RunesEndsWith(input, "ada") || - analysis.RunesEndsWith(input, "iva") || - analysis.RunesEndsWith(input, "ama") { - input[inputLen-1] = 'o' - return input - } - - if analysis.RunesEndsWith(input, "ona") { - input[inputLen-3] = 'ã' - input[inputLen-2] = 'o' - return input[0 : inputLen-1] - } - - if analysis.RunesEndsWith(input, "ora") { - return input[0 : inputLen-1] - } - - if analysis.RunesEndsWith(input, "esa") { - input[inputLen-3] = 'ê' - return input[0 : inputLen-1] - } - - if analysis.RunesEndsWith(input, "na") { - input[inputLen-1] = 'o' - return input - } - } - return input -} - -func PortugueseLightStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewPortugueseLightStemmerFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(LightStemmerName, PortugueseLightStemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt_test.go deleted file mode 100644 index 11afa0c4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/pt/light_stemmer_pt_test.go +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package pt - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestPortugueseLightStemmer(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("doutores"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("doutor"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("doutor"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("doutor"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("homens"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("homem"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("homem"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("homem"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("papéis"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("papel"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("papel"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("papel"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("normais"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("normal"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("normal"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("normal"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("lencóis"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("lencol"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("lencol"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("lencol"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("barris"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("barril"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("barril"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("barril"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("botões"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("bota"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("botão"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("bota"), - }, - }, - }, - // longer - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("o"), - }, - &analysis.Token{ - Term: []byte("debate"), - }, - &analysis.Token{ - Term: []byte("político"), - }, - &analysis.Token{ - Term: []byte("pelo"), - }, - &analysis.Token{ - Term: []byte("menos"), - }, - &analysis.Token{ - Term: []byte("o"), - }, - &analysis.Token{ - Term: []byte("que"), - }, - &analysis.Token{ - Term: []byte("vem"), - }, - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("público"), - }, - &analysis.Token{ - Term: []byte("parece"), - }, - &analysis.Token{ - Term: []byte("de"), - }, - &analysis.Token{ - Term: []byte("modo"), - }, - &analysis.Token{ - Term: []byte("nada"), - }, - &analysis.Token{ - Term: []byte("surpreendente"), - }, - &analysis.Token{ - Term: []byte("restrito"), - }, - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("temas"), - }, - &analysis.Token{ - Term: []byte("menores"), - }, - &analysis.Token{ - Term: []byte("mas"), - }, - &analysis.Token{ - Term: []byte("há"), - }, - &analysis.Token{ - Term: []byte("evidentemente"), - }, - &analysis.Token{ - Term: []byte("grandes"), - }, - &analysis.Token{ - Term: []byte("questões"), - }, - &analysis.Token{ - Term: []byte("em"), - }, - &analysis.Token{ - Term: []byte("jogo"), - }, - &analysis.Token{ - Term: []byte("nas"), - }, - &analysis.Token{ - Term: []byte("eleições"), - }, - &analysis.Token{ - Term: []byte("que"), - }, - &analysis.Token{ - Term: []byte("se"), - }, - &analysis.Token{ - Term: []byte("aproximam"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("o"), - }, - &analysis.Token{ - Term: []byte("debat"), - }, - &analysis.Token{ - Term: []byte("politic"), - }, - &analysis.Token{ - Term: []byte("pelo"), - }, - &analysis.Token{ - Term: []byte("meno"), - }, - &analysis.Token{ - Term: []byte("o"), - }, - &analysis.Token{ - Term: []byte("que"), - }, - &analysis.Token{ - Term: []byte("vem"), - }, - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("public"), - }, - &analysis.Token{ - Term: []byte("parec"), - }, - &analysis.Token{ - Term: []byte("de"), - }, - &analysis.Token{ - Term: []byte("modo"), - }, - &analysis.Token{ - Term: []byte("nada"), - }, - &analysis.Token{ - Term: []byte("surpreendent"), - }, - &analysis.Token{ - Term: []byte("restrit"), - }, - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("tema"), - }, - &analysis.Token{ - Term: []byte("menor"), - }, - &analysis.Token{ - Term: []byte("mas"), - }, - &analysis.Token{ - Term: []byte("há"), - }, - &analysis.Token{ - Term: []byte("evident"), - }, - &analysis.Token{ - Term: []byte("grand"), - }, - &analysis.Token{ - Term: []byte("questa"), - }, - &analysis.Token{ - Term: []byte("em"), - }, - &analysis.Token{ - Term: []byte("jogo"), - }, - &analysis.Token{ - Term: []byte("nas"), - }, - &analysis.Token{ - Term: []byte("eleica"), - }, - &analysis.Token{ - Term: []byte("que"), - }, - &analysis.Token{ - Term: []byte("se"), - }, - &analysis.Token{ - Term: []byte("aproximam"), - }, - }, - }, - } - - cache := registry.NewCache() - filter, err := cache.TokenFilterNamed(LightStemmerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := filter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/pt/stemmer_pt.go b/vendor/github.com/blevesearch/bleve/analysis/language/pt/stemmer_pt.go deleted file mode 100644 index a10c869c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/pt/stemmer_pt.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package pt - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_pt" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("pt") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_filter_pt.go b/vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_filter_pt.go deleted file mode 100644 index e4010b0f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_filter_pt.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package pt - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_words_pt.go b/vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_words_pt.go deleted file mode 100644 index a0d4995f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/pt/stop_words_pt.go +++ /dev/null @@ -1,277 +0,0 @@ -package pt - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_pt" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var PortugueseStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Portuguese stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - - | The following is a ranked list (commonest to rarest) of stopwords - | deriving from a large sample of text. - - | Extra words have been added at the end. - -de | of, from -a | the; to, at; her -o | the; him -que | who, that -e | and -do | de + o -da | de + a -em | in -um | a -para | for - | é from SER -com | with -não | not, no -uma | a -os | the; them -no | em + o -se | himself etc -na | em + a -por | for -mais | more -as | the; them -dos | de + os -como | as, like -mas | but - | foi from SER -ao | a + o -ele | he -das | de + as - | tem from TER -à | a + a -seu | his -sua | her -ou | or - | ser from SER -quando | when -muito | much - | há from HAV -nos | em + os; us -já | already, now - | está from EST -eu | I -também | also -só | only, just -pelo | per + o -pela | per + a -até | up to -isso | that -ela | he -entre | between - | era from SER -depois | after -sem | without -mesmo | same -aos | a + os - | ter from TER -seus | his -quem | whom -nas | em + as -me | me -esse | that -eles | they - | estão from EST -você | you - | tinha from TER - | foram from SER -essa | that -num | em + um -nem | nor -suas | her -meu | my -às | a + as -minha | my - | têm from TER -numa | em + uma -pelos | per + os -elas | they - | havia from HAV - | seja from SER -qual | which - | será from SER -nós | we - | tenho from TER -lhe | to him, her -deles | of them -essas | those -esses | those -pelas | per + as -este | this - | fosse from SER -dele | of him - - | other words. There are many contractions such as naquele = em+aquele, - | mo = me+o, but they are rare. - | Indefinite article plural forms are also rare. - -tu | thou -te | thee -vocês | you (plural) -vos | you -lhes | to them -meus | my -minhas -teu | thy -tua -teus -tuas -nosso | our -nossa -nossos -nossas - -dela | of her -delas | of them - -esta | this -estes | these -estas | these -aquele | that -aquela | that -aqueles | those -aquelas | those -isto | this -aquilo | that - - | forms of estar, to be (not including the infinitive): -estou -está -estamos -estão -estive -esteve -estivemos -estiveram -estava -estávamos -estavam -estivera -estivéramos -esteja -estejamos -estejam -estivesse -estivéssemos -estivessem -estiver -estivermos -estiverem - - | forms of haver, to have (not including the infinitive): -hei -há -havemos -hão -houve -houvemos -houveram -houvera -houvéramos -haja -hajamos -hajam -houvesse -houvéssemos -houvessem -houver -houvermos -houverem -houverei -houverá -houveremos -houverão -houveria -houveríamos -houveriam - - | forms of ser, to be (not including the infinitive): -sou -somos -são -era -éramos -eram -fui -foi -fomos -foram -fora -fôramos -seja -sejamos -sejam -fosse -fôssemos -fossem -for -formos -forem -serei -será -seremos -serão -seria -seríamos -seriam - - | forms of ter, to have (not including the infinitive): -tenho -tem -temos -tém -tinha -tínhamos -tinham -tive -teve -tivemos -tiveram -tivera -tivéramos -tenha -tenhamos -tenham -tivesse -tivéssemos -tivessem -tiver -tivermos -tiverem -terei -terá -teremos -terão -teria -teríamos -teriam -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(PortugueseStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro.go b/vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro.go deleted file mode 100644 index d4d5c1dd..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package ro - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "ro" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopRoFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerRoFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopRoFilter, - stemmerRoFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro_test.go deleted file mode 100644 index 6715b905..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ro/analyzer_ro_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package ro - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestRomanianAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("absenţa"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("absenţ"), - }, - }, - }, - { - input: []byte("absenţi"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("absenţ"), - }, - }, - }, - // stop word - { - input: []byte("îl"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ro/stemmer_ro.go b/vendor/github.com/blevesearch/bleve/analysis/language/ro/stemmer_ro.go deleted file mode 100644 index da18aa66..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ro/stemmer_ro.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package ro - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_ro" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("ro") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_filter_ro.go b/vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_filter_ro.go deleted file mode 100644 index f9259fc5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_filter_ro.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ro - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_words_ro.go b/vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_words_ro.go deleted file mode 100644 index e7d62d41..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ro/stop_words_ro.go +++ /dev/null @@ -1,257 +0,0 @@ -package ro - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_ro" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/ -// ` was changed to ' to allow for literal string - -var RomanianStopWords = []byte(`# This file was created by Jacques Savoy and is distributed under the BSD license. -# See http://members.unine.ch/jacques.savoy/clef/index.html. -# Also see http://www.opensource.org/licenses/bsd-license.html -acea -aceasta -această -aceea -acei -aceia -acel -acela -acele -acelea -acest -acesta -aceste -acestea -aceşti -aceştia -acolo -acum -ai -aia -aibă -aici -al -ăla -ale -alea -ălea -altceva -altcineva -am -ar -are -aş -aşadar -asemenea -asta -ăsta -astăzi -astea -ăstea -ăştia -asupra -aţi -au -avea -avem -aveţi -azi -bine -bucur -bună -ca -că -căci -când -care -cărei -căror -cărui -cât -câte -câţi -către -câtva -ce -cel -ceva -chiar -cînd -cine -cineva -cît -cîte -cîţi -cîtva -contra -cu -cum -cumva -curând -curînd -da -dă -dacă -dar -datorită -de -deci -deja -deoarece -departe -deşi -din -dinaintea -dintr -dintre -drept -după -ea -ei -el -ele -eram -este -eşti -eu -face -fără -fi -fie -fiecare -fii -fim -fiţi -iar -ieri -îi -îl -îmi -împotriva -în -înainte -înaintea -încât -încît -încotro -între -întrucât -întrucît -îţi -la -lângă -le -li -lîngă -lor -lui -mă -mâine -mea -mei -mele -mereu -meu -mi -mine -mult -multă -mulţi -ne -nicăieri -nici -nimeni -nişte -noastră -noastre -noi -noştri -nostru -nu -ori -oricând -oricare -oricât -orice -oricînd -oricine -oricît -oricum -oriunde -până -pe -pentru -peste -pînă -poate -pot -prea -prima -primul -prin -printr -sa -să -săi -sale -sau -său -se -şi -sînt -sîntem -sînteţi -spre -sub -sunt -suntem -sunteţi -ta -tăi -tale -tău -te -ţi -ţie -tine -toată -toate -tot -toţi -totuşi -tu -un -una -unde -undeva -unei -unele -uneori -unor -vă -vi -voastră -voastre -voi -voştri -vostru -vouă -vreo -vreun -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(RomanianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru.go b/vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru.go deleted file mode 100644 index 2aaff852..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package ru - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "ru" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopRuFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerRuFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopRuFilter, - stemmerRuFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru_test.go deleted file mode 100644 index eba2f1d1..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ru/analyzer_ru_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package ru - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestRussianAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // digits safe - { - input: []byte("text 1000"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("text"), - }, - &analysis.Token{ - Term: []byte("1000"), - }, - }, - }, - { - input: []byte("Вместе с тем о силе электромагнитной энергии имели представление еще"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("вмест"), - }, - &analysis.Token{ - Term: []byte("сил"), - }, - &analysis.Token{ - Term: []byte("электромагнитн"), - }, - &analysis.Token{ - Term: []byte("энерг"), - }, - &analysis.Token{ - Term: []byte("имел"), - }, - &analysis.Token{ - Term: []byte("представлен"), - }, - }, - }, - { - input: []byte("Но знание это хранилось в тайне"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("знан"), - }, - &analysis.Token{ - Term: []byte("эт"), - }, - &analysis.Token{ - Term: []byte("хран"), - }, - &analysis.Token{ - Term: []byte("тайн"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ru/stemmer_ru.go b/vendor/github.com/blevesearch/bleve/analysis/language/ru/stemmer_ru.go deleted file mode 100644 index 899d816e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ru/stemmer_ru.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package ru - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_ru" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("ru") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_filter_ru.go b/vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_filter_ru.go deleted file mode 100644 index a7fe18a2..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_filter_ru.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ru - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_words_ru.go b/vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_words_ru.go deleted file mode 100644 index 0129f48c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/ru/stop_words_ru.go +++ /dev/null @@ -1,267 +0,0 @@ -package ru - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_ru" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var RussianStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | a russian stop word list. comments begin with vertical bar. each stop - | word is at the start of a line. - - | this is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - | letter 'ё' is translated to 'е'. - -и | and -в | in/into -во | alternative form -не | not -что | what/that -он | he -на | on/onto -я | i -с | from -со | alternative form -как | how -а | milder form of 'no' (but) -то | conjunction and form of 'that' -все | all -она | she -так | so, thus -его | him -но | but -да | yes/and -ты | thou -к | towards, by -у | around, chez -же | intensifier particle -вы | you -за | beyond, behind -бы | conditional/subj. particle -по | up to, along -только | only -ее | her -мне | to me -было | it was -вот | here is/are, particle -от | away from -меня | me -еще | still, yet, more -нет | no, there isnt/arent -о | about -из | out of -ему | to him -теперь | now -когда | when -даже | even -ну | so, well -вдруг | suddenly -ли | interrogative particle -если | if -уже | already, but homonym of 'narrower' -или | or -ни | neither -быть | to be -был | he was -него | prepositional form of его -до | up to -вас | you accusative -нибудь | indef. suffix preceded by hyphen -опять | again -уж | already, but homonym of 'adder' -вам | to you -сказал | he said -ведь | particle 'after all' -там | there -потом | then -себя | oneself -ничего | nothing -ей | to her -может | usually with 'быть' as 'maybe' -они | they -тут | here -где | where -есть | there is/are -надо | got to, must -ней | prepositional form of ей -для | for -мы | we -тебя | thee -их | them, their -чем | than -была | she was -сам | self -чтоб | in order to -без | without -будто | as if -человек | man, person, one -чего | genitive form of 'what' -раз | once -тоже | also -себе | to oneself -под | beneath -жизнь | life -будет | will be -ж | short form of intensifer particle 'же' -тогда | then -кто | who -этот | this -говорил | was saying -того | genitive form of 'that' -потому | for that reason -этого | genitive form of 'this' -какой | which -совсем | altogether -ним | prepositional form of 'его', 'они' -здесь | here -этом | prepositional form of 'этот' -один | one -почти | almost -мой | my -тем | instrumental/dative plural of 'тот', 'то' -чтобы | full form of 'in order that' -нее | her (acc.) -кажется | it seems -сейчас | now -были | they were -куда | where to -зачем | why -сказать | to say -всех | all (acc., gen. preposn. plural) -никогда | never -сегодня | today -можно | possible, one can -при | by -наконец | finally -два | two -об | alternative form of 'о', about -другой | another -хоть | even -после | after -над | above -больше | more -тот | that one (masc.) -через | across, in -эти | these -нас | us -про | about -всего | in all, only, of all -них | prepositional form of 'они' (they) -какая | which, feminine -много | lots -разве | interrogative particle -сказала | she said -три | three -эту | this, acc. fem. sing. -моя | my, feminine -впрочем | moreover, besides -хорошо | good -свою | ones own, acc. fem. sing. -этой | oblique form of 'эта', fem. 'this' -перед | in front of -иногда | sometimes -лучше | better -чуть | a little -том | preposn. form of 'that one' -нельзя | one must not -такой | such a one -им | to them -более | more -всегда | always -конечно | of course -всю | acc. fem. sing of 'all' -между | between - - - | b: some paradigms - | - | personal pronouns - | - | я меня мне мной [мною] - | ты тебя тебе тобой [тобою] - | он его ему им [него, нему, ним] - | она ее эи ею [нее, нэи, нею] - | оно его ему им [него, нему, ним] - | - | мы нас нам нами - | вы вас вам вами - | они их им ими [них, ним, ними] - | - | себя себе собой [собою] - | - | demonstrative pronouns: этот (this), тот (that) - | - | этот эта это эти - | этого эты это эти - | этого этой этого этих - | этому этой этому этим - | этим этой этим [этою] этими - | этом этой этом этих - | - | тот та то те - | того ту то те - | того той того тех - | тому той тому тем - | тем той тем [тою] теми - | том той том тех - | - | determinative pronouns - | - | (a) весь (all) - | - | весь вся все все - | всего всю все все - | всего всей всего всех - | всему всей всему всем - | всем всей всем [всею] всеми - | всем всей всем всех - | - | (b) сам (himself etc) - | - | сам сама само сами - | самого саму само самих - | самого самой самого самих - | самому самой самому самим - | самим самой самим [самою] самими - | самом самой самом самих - | - | stems of verbs 'to be', 'to have', 'to do' and modal - | - | быть бы буд быв есть суть - | име - | дел - | мог мож мочь - | уме - | хоч хот - | долж - | можн - | нужн - | нельзя - -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(RussianStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv.go b/vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv.go deleted file mode 100644 index 252b1745..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package sv - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "sv" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopSvFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerSvFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopSvFilter, - stemmerSvFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv_test.go deleted file mode 100644 index 4b969572..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/sv/analyzer_sv_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package sv - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestSwedishAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("jaktkarlarne"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("jaktkarl"), - }, - }, - }, - { - input: []byte("jaktkarlens"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("jaktkarl"), - }, - }, - }, - // stop word - { - input: []byte("och"), - output: analysis.TokenStream{}, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/sv/stemmer_sv.go b/vendor/github.com/blevesearch/bleve/analysis/language/sv/stemmer_sv.go deleted file mode 100644 index 67661307..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/sv/stemmer_sv.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package sv - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_sv" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("sv") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_filter_sv.go b/vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_filter_sv.go deleted file mode 100644 index e8930aac..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_filter_sv.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package sv - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_words_sv.go b/vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_words_sv.go deleted file mode 100644 index b4022fd9..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/sv/stop_words_sv.go +++ /dev/null @@ -1,157 +0,0 @@ -package sv - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_sv" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var SwedishStopWords = []byte(` | From svn.tartarus.org/snowball/trunk/website/algorithms/swedish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - | - | NOTE: To use this file with StopFilterFactory, you must specify format="snowball" - - | A Swedish stop word list. Comments begin with vertical bar. Each stop - | word is at the start of a line. - - | This is a ranked list (commonest to rarest) of stopwords derived from - | a large text sample. - - | Swedish stop words occasionally exhibit homonym clashes. For example - | så = so, but also seed. These are indicated clearly below. - -och | and -det | it, this/that -att | to (with infinitive) -i | in, at -en | a -jag | I -hon | she -som | who, that -han | he -på | on -den | it, this/that -med | with -var | where, each -sig | him(self) etc -för | for -så | so (also: seed) -till | to -är | is -men | but -ett | a -om | if; around, about -hade | had -de | they, these/those -av | of -icke | not, no -mig | me -du | you -henne | her -då | then, when -sin | his -nu | now -har | have -inte | inte någon = no one -hans | his -honom | him -skulle | 'sake' -hennes | her -där | there -min | my -man | one (pronoun) -ej | nor -vid | at, by, on (also: vast) -kunde | could -något | some etc -från | from, off -ut | out -när | when -efter | after, behind -upp | up -vi | we -dem | them -vara | be -vad | what -över | over -än | than -dig | you -kan | can -sina | his -här | here -ha | have -mot | towards -alla | all -under | under (also: wonder) -någon | some etc -eller | or (else) -allt | all -mycket | much -sedan | since -ju | why -denna | this/that -själv | myself, yourself etc -detta | this/that -åt | to -utan | without -varit | was -hur | how -ingen | no -mitt | my -ni | you -bli | to be, become -blev | from bli -oss | us -din | thy -dessa | these/those -några | some etc -deras | their -blir | from bli -mina | my -samma | (the) same -vilken | who, that -er | you, your -sådan | such a -vår | our -blivit | from bli -dess | its -inom | within -mellan | between -sådant | such a -varför | why -varje | each -vilka | who, that -ditt | thy -vem | who -vilket | who, that -sitta | his -sådana | such a -vart | each -dina | thy -vars | whose -vårt | our -våra | our -ert | your -era | your -vilkas | whose - -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(SwedishStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th.go b/vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th.go deleted file mode 100644 index a824954a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package th - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" -) - -const AnalyzerName = "th" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - unicodeTokenizer, err := cache.TokenizerNamed(TokenizerName) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopThFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: unicodeTokenizer, - TokenFilters: []analysis.TokenFilter{ - toLowerFilter, - stopThFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th_test.go deleted file mode 100644 index 4d62e35f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/th/analyzer_th_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package th - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -// tried to adapt these from the lucene tests, most of which either -// use the empty stop dictionary or the english one. - -func TestThaiAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stop words - { - input: []byte("การที่ได้ต้องแสดงว่างานดี"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("แสดง"), - Position: 5, - Start: 39, - End: 51, - }, - &analysis.Token{ - Term: []byte("งาน"), - Position: 7, - Start: 60, - End: 69, - }, - &analysis.Token{ - Term: []byte("ดี"), - Position: 8, - Start: 69, - End: 75, - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %v, got %v", test.output, actual) - } - } -} - -func TestThaiAnalyzerWihtoutOffsets(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stop words - { - input: []byte("บริษัทชื่อ XY&Z - คุยกับ xyz@demo.com"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("บริษัท"), - }, - &analysis.Token{ - Term: []byte("ชื่อ"), - }, - &analysis.Token{ - Term: []byte("xy"), - }, - &analysis.Token{ - Term: []byte("z"), - }, - &analysis.Token{ - Term: []byte("คุย"), - }, - &analysis.Token{ - Term: []byte("xyz"), - }, - &analysis.Token{ - Term: []byte("demo.com"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Errorf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/th/stop_filter_th.go b/vendor/github.com/blevesearch/bleve/analysis/language/th/stop_filter_th.go deleted file mode 100644 index b60d1773..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/th/stop_filter_th.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package th - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/th/stop_words_th.go b/vendor/github.com/blevesearch/bleve/analysis/language/th/stop_words_th.go deleted file mode 100644 index e99b10d1..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/th/stop_words_th.go +++ /dev/null @@ -1,143 +0,0 @@ -package th - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_th" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var ThaiStopWords = []byte(`# Thai stopwords from: -# "Opinion Detection in Thai Political News Columns -# Based on Subjectivity Analysis" -# Khampol Sukhum, Supot Nitsuwat, and Choochart Haruechaiyasak -ไว้ -ไม่ -ไป -ได้ -ให้ -ใน -โดย -แห่ง -แล้ว -และ -แรก -แบบ -แต่ -เอง -เห็น -เลย -เริ่ม -เรา -เมื่อ -เพื่อ -เพราะ -เป็นการ -เป็น -เปิดเผย -เปิด -เนื่องจาก -เดียวกัน -เดียว -เช่น -เฉพาะ -เคย -เข้า -เขา -อีก -อาจ -อะไร -ออก -อย่าง -อยู่ -อยาก -หาก -หลาย -หลังจาก -หลัง -หรือ -หนึ่ง -ส่วน -ส่ง -สุด -สําหรับ -ว่า -วัน -ลง -ร่วม -ราย -รับ -ระหว่าง -รวม -ยัง -มี -มาก -มา -พร้อม -พบ -ผ่าน -ผล -บาง -น่า -นี้ -นํา -นั้น -นัก -นอกจาก -ทุก -ที่สุด -ที่ -ทําให้ -ทํา -ทาง -ทั้งนี้ -ทั้ง -ถ้า -ถูก -ถึง -ต้อง -ต่างๆ -ต่าง -ต่อ -ตาม -ตั้งแต่ -ตั้ง -ด้าน -ด้วย -ดัง -ซึ่ง -ช่วง -จึง -จาก -จัด -จะ -คือ -ความ -ครั้ง -คง -ขึ้น -ของ -ขอ -ขณะ -ก่อน -ก็ -การ -กับ -กัน -กว่า -กล่าว -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(ThaiStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/th/unicode_tokenizer_th.go b/vendor/github.com/blevesearch/bleve/analysis/language/th/unicode_tokenizer_th.go deleted file mode 100644 index 3e77382a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/th/unicode_tokenizer_th.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package th - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" - "github.com/blevesearch/bleve/registry" -) - -const TokenizerName = "icu_th" - -func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - return icu.NewUnicodeWordBoundaryCustomLocaleTokenizer("th_TH"), nil -} - -func init() { - registry.RegisterTokenizer(TokenizerName, TokenizerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr.go b/vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr.go deleted file mode 100644 index 8cc06c84..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package tr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter" - "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) - -const AnalyzerName = "tr" - -func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) { - icuTokenizer, err := cache.TokenizerNamed(icu.Name) - if err != nil { - return nil, err - } - aposFilter, err := cache.TokenFilterNamed(apostrophe_filter.Name) - if err != nil { - return nil, err - } - toLowerFilter, err := cache.TokenFilterNamed(lower_case_filter.Name) - if err != nil { - return nil, err - } - stopTrFilter, err := cache.TokenFilterNamed(StopName) - if err != nil { - return nil, err - } - stemmerTrFilter, err := cache.TokenFilterNamed(StemmerName) - if err != nil { - return nil, err - } - rv := analysis.Analyzer{ - Tokenizer: icuTokenizer, - TokenFilters: []analysis.TokenFilter{ - aposFilter, - toLowerFilter, - stopTrFilter, - stemmerTrFilter, - }, - } - return &rv, nil -} - -func init() { - registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr_test.go b/vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr_test.go deleted file mode 100644 index 5b5b1131..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/tr/analyzer_tr_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full -// +build icu full - -package tr - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -func TestTurkishAnalyzer(t *testing.T) { - tests := []struct { - input []byte - output analysis.TokenStream - }{ - // stemming - { - input: []byte("ağacı"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ağaç"), - }, - }, - }, - { - input: []byte("ağaç"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ağaç"), - }, - }, - }, - // stop word - { - input: []byte("dolayı"), - output: analysis.TokenStream{}, - }, - // apostrophes - { - input: []byte("Kıbrıs'ta"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("kıbrıs"), - }, - }, - }, - { - input: []byte("Van Gölü'ne"), - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("van"), - }, - &analysis.Token{ - Term: []byte("göl"), - }, - }, - }, - } - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed(AnalyzerName) - if err != nil { - t.Fatal(err) - } - for _, test := range tests { - actual := analyzer.Analyze(test.input) - if len(actual) != len(test.output) { - t.Fatalf("expected length: %d, got %d", len(test.output), len(actual)) - } - for i, tok := range actual { - if !reflect.DeepEqual(tok.Term, test.output[i].Term) { - t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/tr/stemmer_tr.go b/vendor/github.com/blevesearch/bleve/analysis/language/tr/stemmer_tr.go deleted file mode 100644 index 464296ca..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/tr/stemmer_tr.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package tr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - "github.com/blevesearch/bleve/registry" -) - -const StemmerName = "stemmer_tr" - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return stemmer_filter.NewStemmerFilter("tr") -} - -func init() { - registry.RegisterTokenFilter(StemmerName, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_filter_tr.go b/vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_filter_tr.go deleted file mode 100644 index f8ab8412..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_filter_tr.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package tr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - "github.com/blevesearch/bleve/registry" -) - -func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - tokenMap, err := cache.TokenMapNamed(StopName) - if err != nil { - return nil, err - } - return stop_tokens_filter.NewStopTokensFilter(tokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_words_tr.go b/vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_words_tr.go deleted file mode 100644 index f96fb07e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/language/tr/stop_words_tr.go +++ /dev/null @@ -1,236 +0,0 @@ -package tr - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const StopName = "stop_tr" - -// this content was obtained from: -// lucene-4.7.2/analysis/common/src/resources/org/apache/lucene/analysis/snowball/ -// ` was changed to ' to allow for literal string - -var TurkishStopWords = []byte(`# Turkish stopwords from LUCENE-559 -# merged with the list from "Information Retrieval on Turkish Texts" -# (http://www.users.muohio.edu/canf/papers/JASIST2008offPrint.pdf) -acaba -altmış -altı -ama -ancak -arada -aslında -ayrıca -bana -bazı -belki -ben -benden -beni -benim -beri -beş -bile -bin -bir -birçok -biri -birkaç -birkez -birşey -birşeyi -biz -bize -bizden -bizi -bizim -böyle -böylece -bu -buna -bunda -bundan -bunlar -bunları -bunların -bunu -bunun -burada -çok -çünkü -da -daha -dahi -de -defa -değil -diğer -diye -doksan -dokuz -dolayı -dolayısıyla -dört -edecek -eden -ederek -edilecek -ediliyor -edilmesi -ediyor -eğer -elli -en -etmesi -etti -ettiği -ettiğini -gibi -göre -halen -hangi -hatta -hem -henüz -hep -hepsi -her -herhangi -herkesin -hiç -hiçbir -için -iki -ile -ilgili -ise -işte -itibaren -itibariyle -kadar -karşın -katrilyon -kendi -kendilerine -kendini -kendisi -kendisine -kendisini -kez -ki -kim -kimden -kime -kimi -kimse -kırk -milyar -milyon -mu -mü -mı -nasıl -ne -neden -nedenle -nerde -nerede -nereye -niye -niçin -o -olan -olarak -oldu -olduğu -olduğunu -olduklarını -olmadı -olmadığı -olmak -olması -olmayan -olmaz -olsa -olsun -olup -olur -olursa -oluyor -on -ona -ondan -onlar -onlardan -onları -onların -onu -onun -otuz -oysa -öyle -pek -rağmen -sadece -sanki -sekiz -seksen -sen -senden -seni -senin -siz -sizden -sizi -sizin -şey -şeyden -şeyi -şeyler -şöyle -şu -şuna -şunda -şundan -şunları -şunu -tarafından -trilyon -tüm -üç -üzere -var -vardı -ve -veya -ya -yani -yapacak -yapılan -yapılması -yapıyor -yapmak -yaptı -yaptığı -yaptığını -yaptıkları -yedi -yerine -yetmiş -yine -yirmi -yoksa -yüz -zaten -`) - -func TokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - err := rv.LoadBytes(TurkishStopWords) - return rv, err -} - -func init() { - registry.RegisterTokenMap(StopName, TokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter/lower_case_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go similarity index 61% rename from vendor/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter/lower_case_filter.go rename to vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go index 6d63d6d3..adb740c3 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter/lower_case_filter.go +++ b/vendor/github.com/blevesearch/bleve/analysis/token/lowercase/lowercase.go @@ -1,13 +1,20 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package lower_case_filter +// Package lowercase implements a TokenFilter which converts +// tokens to lower case according to unicode rules. +package lowercase import ( "bytes" @@ -18,6 +25,7 @@ import ( "github.com/blevesearch/bleve/registry" ) +// Name is the name used to register LowerCaseFilter in the bleve registry const Name = "to_lower" type LowerCaseFilter struct { @@ -56,7 +64,24 @@ func toLowerDeferredCopy(s []byte) []byte { if r >= utf8.RuneSelf { r, wid = utf8.DecodeRune(s[i:]) } + l := unicode.ToLower(r) + + // If the rune is already lowercased, just move to the + // next rune. + if l == r { + i += wid + j += wid + continue + } + + // Handles the Unicode edge-case where the last + // rune in a word on the greek Σ needs to be converted + // differently. + if l == 'σ' && i+2 == len(s) { + l = 'ς' + } + lwid := utf8.RuneLen(l) if lwid > wid { // utf-8 encoded replacement is wider diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/porter/porter.go b/vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go similarity index 50% rename from vendor/github.com/blevesearch/bleve/analysis/token_filters/porter/porter.go rename to vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go index 75929385..4cd08d99 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/porter/porter.go +++ b/vendor/github.com/blevesearch/bleve/analysis/token/porter/porter.go @@ -1,15 +1,22 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package porter import ( + "bytes" + "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/registry" @@ -29,8 +36,9 @@ func (s *PorterStemmer) Filter(input analysis.TokenStream) analysis.TokenStream for _, token := range input { // if it is not a protected keyword, stem it if !token.KeyWord { - stemmed := porterstemmer.StemString(string(token.Term)) - token.Term = []byte(stemmed) + termRunes := bytes.Runes(token.Term) + stemmedRunes := porterstemmer.StemWithoutLowerCasing(termRunes) + token.Term = analysis.BuildTermFromRunes(stemmedRunes) } } return input diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter/stop_tokens_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go similarity index 56% rename from vendor/github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter/stop_tokens_filter.go rename to vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go index de3970da..cca2d8e0 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter/stop_tokens_filter.go +++ b/vendor/github.com/blevesearch/bleve/analysis/token/stop/stop.go @@ -1,13 +1,25 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package stop_tokens_filter +// Package stop implements a TokenFilter removing tokens found in +// a TokenMap. +// +// It constructor takes the following arguments: +// +// "stop_token_map" (string): the name of the token map identifying tokens to +// remove. +package stop import ( "fmt" @@ -29,16 +41,16 @@ func NewStopTokensFilter(stopTokens analysis.TokenMap) *StopTokensFilter { } func (f *StopTokensFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - rv := make(analysis.TokenStream, 0, len(input)) - + j := 0 for _, token := range input { _, isStopToken := f.stopTokens[string(token.Term)] if !isStopToken { - rv = append(rv, token) + input[j] = token + j++ } } - return rv + return input[:j] } func StopTokensFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter.go deleted file mode 100644 index 853db823..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package apostrophe_filter - -import ( - "bytes" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "apostrophe" - -const RightSingleQoutationMark = "’" -const Apostrophe = "'" -const Apostrophes = Apostrophe + RightSingleQoutationMark - -type ApostropheFilter struct{} - -func NewApostropheFilter() *ApostropheFilter { - return &ApostropheFilter{} -} - -func (s *ApostropheFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - firstApostrophe := bytes.IndexAny(token.Term, Apostrophes) - if firstApostrophe >= 0 { - // found an apostrophe - token.Term = token.Term[0:firstApostrophe] - } - } - - return input -} - -func ApostropheFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewApostropheFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(Name, ApostropheFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter_test.go deleted file mode 100644 index c9391477..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter/apostrophe_filter_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package apostrophe_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestApostropheFilter(t *testing.T) { - - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Türkiye'de"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Türkiye"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("2003'te"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("2003"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Van"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Van"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Gölü'nü"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Gölü"), - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("gördüm"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("gördüm"), - }, - }, - }, - } - - for _, test := range tests { - apostropheFilter := NewApostropheFilter() - actual := apostropheFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/README.md b/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/README.md deleted file mode 100644 index 64d71710..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# cld2 token filter - -A bleve token filter which passes the text of each token and passes it to the cld2 library. The library determines what it thinks the language most likely is. The ISO-639 language code replaces the token term. - -In normal usage, you use this with the "single" tokenizer, so there is only one input token. Further, you should precede it with the "to_lower" filter so that the input term is in all lower-case unicode characters. - -# Building - -1. Acquire the source to cld2 in this directory. - - $ svn checkout -r 167 http://cld2.googlecode.com/svn/trunk/ cld2-read-only - -2. Build cld2 - - As dynamic library - - $ cd cld2-read-only/internal/ - $ ./compile_libs.sh - $ cp *.so /usr/local/lib - $ cd ../.. - - Or static library - - $ ./compile_cld2.sh - $ cp *.a /usr/local/lib - -3. Run the unit tests - - $ go test -v - === RUN TestCld2Filter - --- PASS: TestCld2Filter (0.00 seconds) - PASS - ok github.com/couchbaselabs/bleve/analysis/token_filters/cld2 0.033s diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.cc b/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.cc deleted file mode 100644 index cb116715..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.cc +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. -#include -#include -#include -#include -#include "cld2_filter.h" -#include "cld2-read-only/public/compact_lang_det.h" - -const char* DetectLang(const char *buffer) { - - bool is_plain_text = true; - CLD2::CLDHints cldhints = {NULL, NULL, 0, CLD2::UNKNOWN_LANGUAGE}; - bool allow_extended_lang = true; - int flags = 0; - CLD2::Language language3[3]; - int percent3[3]; - double normalized_score3[3]; - CLD2::ResultChunkVector resultchunkvector; - int text_bytes; - bool is_reliable; - - CLD2::Language summary_lang = CLD2::UNKNOWN_LANGUAGE; - - summary_lang = CLD2::ExtDetectLanguageSummary(buffer, - strlen(buffer), - is_plain_text, - &cldhints, - flags, - language3, - percent3, - normalized_score3, - &resultchunkvector, - &text_bytes, - &is_reliable); - - return CLD2::LanguageCode(summary_lang); -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.go deleted file mode 100644 index 4349ef8a..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build cld2 full - -package cld2 - -// #cgo LDFLAGS: -lcld2_full -// #include "cld2_filter.h" -// #include -import "C" - -import ( - "unsafe" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "detect_lang" - -type Cld2Filter struct { -} - -func NewCld2Filter() *Cld2Filter { - return &Cld2Filter{} -} - -func (f *Cld2Filter) Filter(input analysis.TokenStream) analysis.TokenStream { - rv := make(analysis.TokenStream, 0, len(input)) - - offset := 0 - for _, token := range input { - var err error - token.Term, err = f.detectLanguage(token.Term) - if err != nil { - token.Term = []byte("error") - } - token.Start = offset - token.End = token.Start + len(token.Term) - token.Type = analysis.AlphaNumeric - rv = append(rv, token) - offset = token.End + 1 - } - - return rv -} - -func (f *Cld2Filter) detectLanguage(input []byte) ([]byte, error) { - cstr := C.CString(string(input)) - res := C.DetectLang(cstr) - return C.GoBytes(unsafe.Pointer(res), C.int(C.strlen(res))), nil -} - -func Cld2FilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewCld2Filter(), nil -} - -func init() { - registry.RegisterTokenFilter(Name, Cld2FilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.h b/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.h deleted file mode 100644 index 8f692986..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. -#ifdef __cplusplus -extern "C" { -#endif - -const char* DetectLang(const char *buffer); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter_test.go deleted file mode 100644 index 00951090..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/cld2_filter_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build cld2 full - -package cld2 - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestCld2Filter(t *testing.T) { - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the quick brown fox"), - Start: 0, - End: 19, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("en"), - Start: 0, - End: 2, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こんにちは世界"), - Start: 0, - End: 21, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ja"), - Start: 0, - End: 2, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), - Start: 0, - End: 72, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("th"), - Start: 0, - End: 2, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("مرحبا، العالم!"), - Start: 0, - End: 26, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ar"), - Start: 0, - End: 2, - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - } - - filter := NewCld2Filter() - for _, test := range tests { - res := filter.Filter(test.input) - if !reflect.DeepEqual(res, test.output) { - t.Errorf("expected:") - for _, token := range test.output { - t.Errorf("%#v - %s", token, token.Term) - } - t.Errorf("got:") - for _, token := range res { - t.Errorf("%#v - %s", token, token.Term) - } - } - } - -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/compile_cld2.sh b/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/compile_cld2.sh deleted file mode 100644 index 71840bbe..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/cld2/compile_cld2.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -SRC="cldutil cldutil_shared compact_lang_det compact_lang_det_hint_code compact_lang_det_impl debug fixunicodevalue generated_entities generated_language generated_ulscript getonescriptspan lang_script offsetmap scoreonescriptspan tote utf8statetable cld_generated_cjk_uni_prop_80 cld2_generated_cjk_compatible cld_generated_cjk_delta_bi_32 generated_distinct_bi_0 cld2_generated_quad0122 cld2_generated_deltaocta0122 cld2_generated_distinctocta0122 cld_generated_score_quad_octa_0122"; -OBJ=""; -for f in ${SRC}; do - g++ -c -fPIC -O2 -m64 -o "cld2-read-only/internal/${f}.o" "cld2-read-only/internal/${f}.cc"; - OBJ="${OBJ} cld2-read-only/internal/${f}.o"; -done; - -ar rcs libcld2_full.a ${OBJ}; diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict.go deleted file mode 100644 index f4c7a049..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package compound - -import ( - "bytes" - "fmt" - "unicode/utf8" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "dict_compound" - -const defaultMinWordSize = 5 -const defaultMinSubWordSize = 2 -const defaultMaxSubWordSize = 15 -const defaultOnlyLongestMatch = false - -type DictionaryCompoundFilter struct { - dict analysis.TokenMap - minWordSize int - minSubWordSize int - maxSubWordSize int - onlyLongestMatch bool -} - -func NewDictionaryCompoundFilter(dict analysis.TokenMap, minWordSize, minSubWordSize, maxSubWordSize int, onlyLongestMatch bool) *DictionaryCompoundFilter { - return &DictionaryCompoundFilter{ - dict: dict, - minWordSize: minWordSize, - minSubWordSize: minSubWordSize, - maxSubWordSize: maxSubWordSize, - onlyLongestMatch: onlyLongestMatch, - } -} - -func (f *DictionaryCompoundFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - rv := make(analysis.TokenStream, 0, len(input)) - - for _, token := range input { - rv = append(rv, token) - tokenLen := utf8.RuneCount(token.Term) - if tokenLen >= f.minWordSize { - newtokens := f.decompose(token) - for _, newtoken := range newtokens { - rv = append(rv, newtoken) - } - } - } - - return rv -} - -func (f *DictionaryCompoundFilter) decompose(token *analysis.Token) []*analysis.Token { - runes := bytes.Runes(token.Term) - rv := make([]*analysis.Token, 0) - rlen := len(runes) - for i := 0; i <= (rlen - f.minSubWordSize); i++ { - var longestMatchToken *analysis.Token - for j := f.minSubWordSize; j <= f.maxSubWordSize; j++ { - if i+j > rlen { - break - } - _, inDict := f.dict[string(runes[i:i+j])] - if inDict { - newtoken := analysis.Token{ - Term: []byte(string(runes[i : i+j])), - Position: token.Position, - Start: token.Start + i, - End: token.Start + i + j, - Type: token.Type, - KeyWord: token.KeyWord, - } - if f.onlyLongestMatch { - if longestMatchToken == nil || utf8.RuneCount(longestMatchToken.Term) < j { - longestMatchToken = &newtoken - } - } else { - rv = append(rv, &newtoken) - } - } - } - if f.onlyLongestMatch && longestMatchToken != nil { - rv = append(rv, longestMatchToken) - } - } - return rv -} - -func DictionaryCompoundFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - - minWordSize := defaultMinWordSize - minSubWordSize := defaultMinSubWordSize - maxSubWordSize := defaultMaxSubWordSize - onlyLongestMatch := defaultOnlyLongestMatch - - minVal, ok := config["min_word_size"].(float64) - if ok { - minWordSize = int(minVal) - } - minSubVal, ok := config["min_subword_size"].(float64) - if ok { - minSubWordSize = int(minSubVal) - } - maxSubVal, ok := config["max_subword_size"].(float64) - if ok { - maxSubWordSize = int(maxSubVal) - } - onlyVal, ok := config["only_longest_match"].(bool) - if ok { - onlyLongestMatch = onlyVal - } - - dictTokenMapName, ok := config["dict_token_map"].(string) - if !ok { - return nil, fmt.Errorf("must specify dict_token_map") - } - dictTokenMap, err := cache.TokenMapNamed(dictTokenMapName) - if err != nil { - return nil, fmt.Errorf("error building dict compound words filter: %v", err) - } - return NewDictionaryCompoundFilter(dictTokenMap, minWordSize, minSubWordSize, maxSubWordSize, onlyLongestMatch), nil -} - -func init() { - registry.RegisterTokenFilter(Name, DictionaryCompoundFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict_test.go deleted file mode 100644 index 9a8d1ada..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/compound/dict_test.go +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package compound - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_map" - "github.com/blevesearch/bleve/registry" -) - -func TestStopWordsFilter(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("i"), - Start: 0, - End: 1, - Position: 1, - }, - &analysis.Token{ - Term: []byte("like"), - Start: 2, - End: 6, - Position: 2, - }, - &analysis.Token{ - Term: []byte("to"), - Start: 7, - End: 9, - Position: 3, - }, - &analysis.Token{ - Term: []byte("play"), - Start: 10, - End: 14, - Position: 4, - }, - &analysis.Token{ - Term: []byte("softball"), - Start: 15, - End: 23, - Position: 5, - }, - } - - expectedTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("i"), - Start: 0, - End: 1, - Position: 1, - }, - &analysis.Token{ - Term: []byte("like"), - Start: 2, - End: 6, - Position: 2, - }, - &analysis.Token{ - Term: []byte("to"), - Start: 7, - End: 9, - Position: 3, - }, - &analysis.Token{ - Term: []byte("play"), - Start: 10, - End: 14, - Position: 4, - }, - &analysis.Token{ - Term: []byte("softball"), - Start: 15, - End: 23, - Position: 5, - }, - &analysis.Token{ - Term: []byte("soft"), - Start: 15, - End: 19, - Position: 5, - }, - &analysis.Token{ - Term: []byte("ball"), - Start: 19, - End: 23, - Position: 5, - }, - } - - cache := registry.NewCache() - dictListConfig := map[string]interface{}{ - "type": token_map.Name, - "tokens": []interface{}{"factor", "soft", "ball", "team"}, - } - _, err := cache.DefineTokenMap("dict_test", dictListConfig) - if err != nil { - t.Fatal(err) - } - - dictConfig := map[string]interface{}{ - "type": "dict_compound", - "dict_token_map": "dict_test", - } - dictFilter, err := cache.DefineTokenFilter("dict_test", dictConfig) - if err != nil { - t.Fatal(err) - } - - ouputTokenStream := dictFilter.Filter(inputTokenStream) - if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { - t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) - } -} - -func TestStopWordsFilterLongestMatch(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("softestball"), - Start: 0, - End: 11, - Position: 1, - }, - } - - expectedTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("softestball"), - Start: 0, - End: 11, - Position: 1, - }, - &analysis.Token{ - Term: []byte("softest"), - Start: 0, - End: 7, - Position: 1, - }, - &analysis.Token{ - Term: []byte("ball"), - Start: 7, - End: 11, - Position: 1, - }, - } - - cache := registry.NewCache() - dictListConfig := map[string]interface{}{ - "type": token_map.Name, - "tokens": []interface{}{"soft", "softest", "ball"}, - } - _, err := cache.DefineTokenMap("dict_test", dictListConfig) - if err != nil { - t.Fatal(err) - } - - dictConfig := map[string]interface{}{ - "type": "dict_compound", - "dict_token_map": "dict_test", - "only_longest_match": true, - } - dictFilter, err := cache.DefineTokenFilter("dict_test", dictConfig) - if err != nil { - t.Fatal(err) - } - - ouputTokenStream := dictFilter.Filter(inputTokenStream) - if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { - t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter.go deleted file mode 100644 index f370aa9d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package edge_ngram_filter - -import ( - "bytes" - "fmt" - "unicode/utf8" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "edge_ngram" - -type Side bool - -const BACK Side = true -const FRONT Side = false - -type EdgeNgramFilter struct { - back Side - minLength int - maxLength int -} - -func NewEdgeNgramFilter(side Side, minLength, maxLength int) *EdgeNgramFilter { - return &EdgeNgramFilter{ - back: side, - minLength: minLength, - maxLength: maxLength, - } -} - -func (s *EdgeNgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - rv := make(analysis.TokenStream, 0, len(input)) - - for _, token := range input { - runeCount := utf8.RuneCount(token.Term) - runes := bytes.Runes(token.Term) - if s.back { - i := runeCount - // index of the starting rune for this token - for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ { - // build an ngram of this size starting at i - if i-ngramSize > 0 { - ngramTerm := buildTermFromRunes(runes[i-ngramSize : i]) - token := analysis.Token{ - Position: token.Position, - Start: token.Start, - End: token.End, - Type: token.Type, - Term: ngramTerm, - } - rv = append(rv, &token) - } - } - } else { - i := 0 - // index of the starting rune for this token - for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ { - // build an ngram of this size starting at i - if i+ngramSize <= runeCount { - ngramTerm := buildTermFromRunes(runes[i : i+ngramSize]) - token := analysis.Token{ - Position: token.Position, - Start: token.Start, - End: token.End, - Type: token.Type, - Term: ngramTerm, - } - rv = append(rv, &token) - } - } - } - } - - return rv -} - -func buildTermFromRunes(runes []rune) []byte { - rv := make([]byte, 0, len(runes)*4) - for _, r := range runes { - runeBytes := make([]byte, utf8.RuneLen(r)) - utf8.EncodeRune(runeBytes, r) - rv = append(rv, runeBytes...) - } - return rv -} - -func EdgeNgramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - side := FRONT - back, ok := config["back"].(bool) - if ok && back { - side = BACK - } - minVal, ok := config["min"].(float64) - if !ok { - return nil, fmt.Errorf("must specify min") - } - min := int(minVal) - maxVal, ok := config["max"].(float64) - if !ok { - return nil, fmt.Errorf("must specify max") - } - max := int(maxVal) - - return NewEdgeNgramFilter(side, min, max), nil -} - -func init() { - registry.RegisterTokenFilter(Name, EdgeNgramFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter_test.go deleted file mode 100644 index a033b1e4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter/edge_ngram_filter_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package edge_ngram_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestEdgeNgramFilter(t *testing.T) { - - tests := []struct { - side Side - min int - max int - input analysis.TokenStream - output analysis.TokenStream - }{ - { - side: FRONT, - min: 1, - max: 1, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - }, - }, - { - side: BACK, - min: 1, - max: 1, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("e"), - }, - }, - }, - { - side: FRONT, - min: 1, - max: 3, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("ab"), - }, - &analysis.Token{ - Term: []byte("abc"), - }, - }, - }, - { - side: BACK, - min: 1, - max: 3, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("e"), - }, - &analysis.Token{ - Term: []byte("de"), - }, - &analysis.Token{ - Term: []byte("cde"), - }, - }, - }, - { - side: FRONT, - min: 1, - max: 3, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - &analysis.Token{ - Term: []byte("vwxyz"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("ab"), - }, - &analysis.Token{ - Term: []byte("abc"), - }, - &analysis.Token{ - Term: []byte("v"), - }, - &analysis.Token{ - Term: []byte("vw"), - }, - &analysis.Token{ - Term: []byte("vwx"), - }, - }, - }, - } - - for _, test := range tests { - edgeNgramFilter := NewEdgeNgramFilter(test.side, test.min, test.max) - actual := edgeNgramFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter.go deleted file mode 100644 index 61aa6137..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package elision_filter - -import ( - "bytes" - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "elision" - -const RightSingleQoutationMark = "’" -const Apostrophe = "'" - -const Apostrophes = Apostrophe + RightSingleQoutationMark - -type ElisionFilter struct { - articles analysis.TokenMap -} - -func NewElisionFilter(articles analysis.TokenMap) *ElisionFilter { - return &ElisionFilter{ - articles: articles, - } -} - -func (s *ElisionFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - firstApostrophe := bytes.IndexAny(token.Term, Apostrophes) - if firstApostrophe >= 0 { - // found an apostrophe - prefix := token.Term[0:firstApostrophe] - // see if the prefix matches one of the articles - _, articleMatch := s.articles[string(prefix)] - if articleMatch { - token.Term = token.Term[firstApostrophe+1:] - } - } - } - return input -} - -func ElisionFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - articlesTokenMapName, ok := config["articles_token_map"].(string) - if !ok { - return nil, fmt.Errorf("must specify articles_token_map") - } - articlesTokenMap, err := cache.TokenMapNamed(articlesTokenMapName) - if err != nil { - return nil, fmt.Errorf("error building elision filter: %v", err) - } - return NewElisionFilter(articlesTokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(Name, ElisionFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter_test.go deleted file mode 100644 index ba463370..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/elision_filter/elision_filter_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package elision_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_map" - "github.com/blevesearch/bleve/registry" -) - -func TestElisionFilter(t *testing.T) { - - tests := []struct { - input analysis.TokenStream - output analysis.TokenStream - }{ - { - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ar'word"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("word"), - }, - }, - }, - } - - cache := registry.NewCache() - - articleListConfig := map[string]interface{}{ - "type": token_map.Name, - "tokens": []interface{}{"ar"}, - } - _, err := cache.DefineTokenMap("articles_test", articleListConfig) - if err != nil { - t.Fatal(err) - } - - elisionConfig := map[string]interface{}{ - "type": "elision", - "articles_token_map": "articles_test", - } - elisionFilter, err := cache.DefineTokenFilter("elision_test", elisionConfig) - if err != nil { - t.Fatal(err) - } - - for _, test := range tests { - - actual := elisionFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter.go deleted file mode 100644 index 062d2172..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package keyword_filter - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "keyword_marker" - -type KeyWordMarkerFilter struct { - keyWords analysis.TokenMap -} - -func NewKeyWordMarkerFilter(keyWords analysis.TokenMap) *KeyWordMarkerFilter { - return &KeyWordMarkerFilter{ - keyWords: keyWords, - } -} - -func (f *KeyWordMarkerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - word := string(token.Term) - _, isKeyWord := f.keyWords[word] - if isKeyWord { - token.KeyWord = true - } - } - return input -} - -func KeyWordMarkerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - keywordsTokenMapName, ok := config["keywords_token_map"].(string) - if !ok { - return nil, fmt.Errorf("must specify keywords_token_map") - } - keywordsTokenMap, err := cache.TokenMapNamed(keywordsTokenMapName) - if err != nil { - return nil, fmt.Errorf("error building keyword marker filter: %v", err) - } - return NewKeyWordMarkerFilter(keywordsTokenMap), nil -} - -func init() { - registry.RegisterTokenFilter(Name, KeyWordMarkerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter_test.go deleted file mode 100644 index f6d83824..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter/keyword_marker_filter_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package keyword_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestKeyWordMarkerFilter(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("walk"), - }, - &analysis.Token{ - Term: []byte("in"), - }, - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("park"), - }, - } - - expectedTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("walk"), - KeyWord: true, - }, - &analysis.Token{ - Term: []byte("in"), - }, - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("park"), - KeyWord: true, - }, - } - - keyWordsMap := analysis.NewTokenMap() - keyWordsMap.AddToken("walk") - keyWordsMap.AddToken("park") - - filter := NewKeyWordMarkerFilter(keyWordsMap) - ouputTokenStream := filter.Filter(inputTokenStream) - if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { - t.Errorf("expected %#v got %#v", expectedTokenStream[0].KeyWord, ouputTokenStream[0].KeyWord) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter.go deleted file mode 100644 index 118c583c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package length_filter - -import ( - "fmt" - "unicode/utf8" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "length" - -type LengthFilter struct { - min int - max int -} - -func NewLengthFilter(min, max int) *LengthFilter { - return &LengthFilter{ - min: min, - max: max, - } -} - -func (f *LengthFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - rv := make(analysis.TokenStream, 0, len(input)) - - for _, token := range input { - wordLen := utf8.RuneCount(token.Term) - if f.min > 0 && f.min > wordLen { - continue - } - if f.max > 0 && f.max < wordLen { - continue - } - rv = append(rv, token) - } - - return rv -} - -func LengthFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - min := 0 - max := 0 - - minVal, ok := config["min"].(float64) - if ok { - min = int(minVal) - } - maxVal, ok := config["max"].(float64) - if ok { - max = int(maxVal) - } - if min == max && max == 0 { - return nil, fmt.Errorf("either min or max must be non-zero") - } - - return NewLengthFilter(min, max), nil -} - -func init() { - registry.RegisterTokenFilter(Name, LengthFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter_test.go deleted file mode 100644 index 5ac4bb30..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/length_filter/length_filter_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package length_filter - -import ( - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestLengthFilter(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1"), - }, - &analysis.Token{ - Term: []byte("two"), - }, - &analysis.Token{ - Term: []byte("three"), - }, - } - - lengthFilter := NewLengthFilter(3, 4) - ouputTokenStream := lengthFilter.Filter(inputTokenStream) - if len(ouputTokenStream) != 1 { - t.Fatalf("expected 1 output token") - } - if string(ouputTokenStream[0].Term) != "two" { - t.Errorf("expected term `two`, got `%s`", ouputTokenStream[0].Term) - } -} - -func TestLengthFilterNoMax(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1"), - }, - &analysis.Token{ - Term: []byte("two"), - }, - &analysis.Token{ - Term: []byte("three"), - }, - } - - lengthFilter := NewLengthFilter(3, -1) - ouputTokenStream := lengthFilter.Filter(inputTokenStream) - if len(ouputTokenStream) != 2 { - t.Fatalf("expected 2 output token") - } - if string(ouputTokenStream[0].Term) != "two" { - t.Errorf("expected term `two`, got `%s`", ouputTokenStream[0].Term) - } - if string(ouputTokenStream[1].Term) != "three" { - t.Errorf("expected term `three`, got `%s`", ouputTokenStream[0].Term) - } -} - -func TestLengthFilterNoMin(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1"), - }, - &analysis.Token{ - Term: []byte("two"), - }, - &analysis.Token{ - Term: []byte("three"), - }, - } - - lengthFilter := NewLengthFilter(-1, 4) - ouputTokenStream := lengthFilter.Filter(inputTokenStream) - if len(ouputTokenStream) != 2 { - t.Fatalf("expected 2 output token") - } - if string(ouputTokenStream[0].Term) != "1" { - t.Errorf("expected term `1`, got `%s`", ouputTokenStream[0].Term) - } - if string(ouputTokenStream[1].Term) != "two" { - t.Errorf("expected term `two`, got `%s`", ouputTokenStream[0].Term) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter/lower_case_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter/lower_case_filter_test.go deleted file mode 100644 index ccb00ead..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter/lower_case_filter_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package lower_case_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestLowerCaseFilter(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ONE"), - }, - &analysis.Token{ - Term: []byte("two"), - }, - &analysis.Token{ - Term: []byte("ThReE"), - }, - &analysis.Token{ - Term: []byte("steven's"), - }, - // these characters are chosen in particular - // because the utf-8 encoding of the lower-case - // version has a different length - // Rune İ(304) width 2 - Lower i(105) width 1 - // Rune Ⱥ(570) width 2 - Lower ⱥ(11365) width 3 - // Rune Ⱦ(574) width 2 - Lower ⱦ(11366) width 3 - &analysis.Token{ - Term: []byte("İȺȾCAT"), - }, - &analysis.Token{ - Term: []byte("ȺȾCAT"), - }, - } - - expectedTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("one"), - }, - &analysis.Token{ - Term: []byte("two"), - }, - &analysis.Token{ - Term: []byte("three"), - }, - &analysis.Token{ - Term: []byte("steven's"), - }, - &analysis.Token{ - Term: []byte("iⱥⱦcat"), - }, - &analysis.Token{ - Term: []byte("ⱥⱦcat"), - }, - } - - filter := NewLowerCaseFilter() - ouputTokenStream := filter.Filter(inputTokenStream) - if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { - t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) - t.Errorf("expected %s got %s", expectedTokenStream[0].Term, ouputTokenStream[0].Term) - } -} - -func BenchmarkLowerCaseFilter(b *testing.B) { - input := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("A"), - }, - &analysis.Token{ - Term: []byte("boiling"), - }, - &analysis.Token{ - Term: []byte("liquid"), - }, - &analysis.Token{ - Term: []byte("expanding"), - }, - &analysis.Token{ - Term: []byte("vapor"), - }, - &analysis.Token{ - Term: []byte("explosion"), - }, - &analysis.Token{ - Term: []byte("caused"), - }, - &analysis.Token{ - Term: []byte("by"), - }, - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("rupture"), - }, - &analysis.Token{ - Term: []byte("of"), - }, - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("vessel"), - }, - &analysis.Token{ - Term: []byte("containing"), - }, - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("pressurized"), - }, - &analysis.Token{ - Term: []byte("liquid"), - }, - &analysis.Token{ - Term: []byte("above"), - }, - &analysis.Token{ - Term: []byte("its"), - }, - &analysis.Token{ - Term: []byte("boiling"), - }, - &analysis.Token{ - Term: []byte("point"), - }, - &analysis.Token{ - Term: []byte("İȺȾCAT"), - }, - &analysis.Token{ - Term: []byte("ȺȾCAT"), - }, - } - filter := NewLowerCaseFilter() - - b.ResetTimer() - for i := 0; i < b.N; i++ { - filter.Filter(input) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter.go deleted file mode 100644 index d1433e26..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ngram_filter - -import ( - "bytes" - "fmt" - "unicode/utf8" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "ngram" - -type NgramFilter struct { - minLength int - maxLength int -} - -func NewNgramFilter(minLength, maxLength int) *NgramFilter { - return &NgramFilter{ - minLength: minLength, - maxLength: maxLength, - } -} - -func (s *NgramFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - rv := make(analysis.TokenStream, 0, len(input)) - - for _, token := range input { - runeCount := utf8.RuneCount(token.Term) - runes := bytes.Runes(token.Term) - for i := 0; i < runeCount; i++ { - // index of the starting rune for this token - for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ { - // build an ngram of this size starting at i - if i+ngramSize <= runeCount { - ngramTerm := buildTermFromRunes(runes[i : i+ngramSize]) - token := analysis.Token{ - Position: token.Position, - Start: token.Start, - End: token.End, - Type: token.Type, - Term: ngramTerm, - } - rv = append(rv, &token) - } - } - } - } - - return rv -} - -func buildTermFromRunes(runes []rune) []byte { - rv := make([]byte, 0, len(runes)*4) - for _, r := range runes { - runeBytes := make([]byte, utf8.RuneLen(r)) - utf8.EncodeRune(runeBytes, r) - rv = append(rv, runeBytes...) - } - return rv -} - -func NgramFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - minVal, ok := config["min"].(float64) - if !ok { - return nil, fmt.Errorf("must specify min") - } - min := int(minVal) - maxVal, ok := config["max"].(float64) - if !ok { - return nil, fmt.Errorf("must specify max") - } - max := int(maxVal) - - return NewNgramFilter(min, max), nil -} - -func init() { - registry.RegisterTokenFilter(Name, NgramFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter_test.go deleted file mode 100644 index a5a52c50..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/ngram_filter/ngram_filter_test.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ngram_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestNgramFilter(t *testing.T) { - - tests := []struct { - min int - max int - input analysis.TokenStream - output analysis.TokenStream - }{ - { - min: 1, - max: 1, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("b"), - }, - &analysis.Token{ - Term: []byte("c"), - }, - &analysis.Token{ - Term: []byte("d"), - }, - &analysis.Token{ - Term: []byte("e"), - }, - }, - }, - { - min: 2, - max: 2, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ab"), - }, - &analysis.Token{ - Term: []byte("bc"), - }, - &analysis.Token{ - Term: []byte("cd"), - }, - &analysis.Token{ - Term: []byte("de"), - }, - }, - }, - { - min: 1, - max: 3, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("ab"), - }, - &analysis.Token{ - Term: []byte("abc"), - }, - &analysis.Token{ - Term: []byte("b"), - }, - &analysis.Token{ - Term: []byte("bc"), - }, - &analysis.Token{ - Term: []byte("bcd"), - }, - &analysis.Token{ - Term: []byte("c"), - }, - &analysis.Token{ - Term: []byte("cd"), - }, - &analysis.Token{ - Term: []byte("cde"), - }, - &analysis.Token{ - Term: []byte("d"), - }, - &analysis.Token{ - Term: []byte("de"), - }, - &analysis.Token{ - Term: []byte("e"), - }, - }, - }, - } - - for _, test := range tests { - ngramFilter := NewNgramFilter(test.min, test.max) - actual := ngramFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/porter/porter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/porter/porter_test.go deleted file mode 100644 index ffea83f6..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/porter/porter_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package porter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestPorterStemmer(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("walking"), - }, - &analysis.Token{ - Term: []byte("talked"), - }, - &analysis.Token{ - Term: []byte("business"), - }, - &analysis.Token{ - Term: []byte("protected"), - KeyWord: true, - }, - } - - expectedTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("walk"), - }, - &analysis.Token{ - Term: []byte("talk"), - }, - &analysis.Token{ - Term: []byte("busi"), - }, - &analysis.Token{ - Term: []byte("protected"), - KeyWord: true, - }, - } - - filter := NewPorterStemmer() - ouputTokenStream := filter.Filter(inputTokenStream) - if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { - t.Errorf("expected %#v got %#v", expectedTokenStream[3], ouputTokenStream[3]) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle.go deleted file mode 100644 index 90965092..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle.go +++ /dev/null @@ -1,157 +0,0 @@ -package shingle - -import ( - "container/ring" - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "shingle" - -type ShingleFilter struct { - min int - max int - outputOriginal bool - tokenSeparator string - fill string - ring *ring.Ring - itemsInRing int -} - -func NewShingleFilter(min, max int, outputOriginal bool, sep, fill string) *ShingleFilter { - return &ShingleFilter{ - min: min, - max: max, - outputOriginal: outputOriginal, - tokenSeparator: sep, - fill: fill, - ring: ring.New(max), - } -} - -func (s *ShingleFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - rv := make(analysis.TokenStream, 0, len(input)) - - currentPosition := 0 - for _, token := range input { - if s.outputOriginal { - rv = append(rv, token) - } - - // if there are gaps, insert filler tokens - offset := token.Position - currentPosition - for offset > 1 { - fillerToken := analysis.Token{ - Position: 0, - Start: -1, - End: -1, - Type: analysis.AlphaNumeric, - Term: []byte(s.fill), - } - s.ring.Value = &fillerToken - if s.itemsInRing < s.max { - s.itemsInRing++ - } - rv = append(rv, s.shingleCurrentRingState()...) - s.ring = s.ring.Next() - offset-- - } - currentPosition = token.Position - - s.ring.Value = token - if s.itemsInRing < s.max { - s.itemsInRing++ - } - rv = append(rv, s.shingleCurrentRingState()...) - s.ring = s.ring.Next() - - } - - return rv -} - -func (s *ShingleFilter) shingleCurrentRingState() analysis.TokenStream { - rv := make(analysis.TokenStream, 0) - for shingleN := s.min; shingleN <= s.max; shingleN++ { - // if there are enough items in the ring - // to produce a shingle of this size - if s.itemsInRing >= shingleN { - thisShingleRing := s.ring.Move(-(shingleN - 1)) - shingledBytes := make([]byte, 0) - pos := 0 - start := -1 - end := 0 - for i := 0; i < shingleN; i++ { - if i != 0 { - shingledBytes = append(shingledBytes, []byte(s.tokenSeparator)...) - } - curr := thisShingleRing.Value.(*analysis.Token) - if pos == 0 && curr.Position != 0 { - pos = curr.Position - } - if start == -1 && curr.Start != -1 { - start = curr.Start - } - if curr.End != -1 { - end = curr.End - } - shingledBytes = append(shingledBytes, curr.Term...) - thisShingleRing = thisShingleRing.Next() - } - token := analysis.Token{ - Type: analysis.Shingle, - Term: shingledBytes, - } - if pos != 0 { - token.Position = pos - } - if start != -1 { - token.Start = start - } - if end != -1 { - token.End = end - } - rv = append(rv, &token) - } - } - return rv -} - -func ShingleFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - minVal, ok := config["min"].(float64) - if !ok { - return nil, fmt.Errorf("must specify min") - } - min := int(minVal) - maxVal, ok := config["max"].(float64) - if !ok { - return nil, fmt.Errorf("must specify max") - } - max := int(maxVal) - - outputOriginal := false - outVal, ok := config["output_original"].(bool) - if ok { - outputOriginal = outVal - } - - sep := " " - sepVal, ok := config["separator"].(string) - if ok { - sep = sepVal - } - - fill := "_" - fillVal, ok := config["filler"].(string) - if ok { - fill = fillVal - } - - return NewShingleFilter(min, max, outputOriginal, sep, fill), nil -} - -func init() { - registry.RegisterTokenFilter(Name, ShingleFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle_test.go deleted file mode 100644 index d81ac57e..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/shingle/shingle_test.go +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package shingle - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestShingleFilter(t *testing.T) { - - tests := []struct { - min int - max int - outputOriginal bool - separator string - filler string - input analysis.TokenStream - output analysis.TokenStream - }{ - { - min: 2, - max: 2, - outputOriginal: false, - separator: " ", - filler: "_", - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("quick"), - }, - &analysis.Token{ - Term: []byte("brown"), - }, - &analysis.Token{ - Term: []byte("fox"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the quick"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("quick brown"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("brown fox"), - Type: analysis.Shingle, - }, - }, - }, - { - min: 3, - max: 3, - outputOriginal: false, - separator: " ", - filler: "_", - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("quick"), - }, - &analysis.Token{ - Term: []byte("brown"), - }, - &analysis.Token{ - Term: []byte("fox"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the quick brown"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("quick brown fox"), - Type: analysis.Shingle, - }, - }, - }, - { - min: 2, - max: 3, - outputOriginal: false, - separator: " ", - filler: "_", - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("quick"), - }, - &analysis.Token{ - Term: []byte("brown"), - }, - &analysis.Token{ - Term: []byte("fox"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the quick"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("quick brown"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("the quick brown"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("brown fox"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("quick brown fox"), - Type: analysis.Shingle, - }, - }, - }, - { - min: 3, - max: 3, - outputOriginal: false, - separator: " ", - filler: "_", - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ugly"), - Position: 1, - }, - &analysis.Token{ - Term: []byte("quick"), - Position: 3, - }, - &analysis.Token{ - Term: []byte("brown"), - Position: 4, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ugly _ quick"), - Type: analysis.Shingle, - Position: 1, - }, - &analysis.Token{ - Term: []byte("_ quick brown"), - Type: analysis.Shingle, - Position: 3, - }, - }, - }, - { - min: 1, - max: 5, - outputOriginal: false, - separator: " ", - filler: "_", - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("test"), - Position: 1, - }, - &analysis.Token{ - Term: []byte("text"), - Position: 2, - }, - // token 3 removed by stop filter - &analysis.Token{ - Term: []byte("see"), - Position: 4, - }, - &analysis.Token{ - Term: []byte("shingles"), - Position: 5, - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("test"), - Type: analysis.Shingle, - Position: 1, - }, - &analysis.Token{ - Term: []byte("text"), - Type: analysis.Shingle, - Position: 2, - }, - &analysis.Token{ - Term: []byte("test text"), - Type: analysis.Shingle, - Position: 1, - }, - &analysis.Token{ - Term: []byte("_"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("text _"), - Type: analysis.Shingle, - Position: 2, - }, - &analysis.Token{ - Term: []byte("test text _"), - Type: analysis.Shingle, - Position: 1, - }, - &analysis.Token{ - Term: []byte("see"), - Type: analysis.Shingle, - Position: 4, - }, - &analysis.Token{ - Term: []byte("_ see"), - Type: analysis.Shingle, - Position: 4, - }, - &analysis.Token{ - Term: []byte("text _ see"), - Type: analysis.Shingle, - Position: 2, - }, - &analysis.Token{ - Term: []byte("test text _ see"), - Type: analysis.Shingle, - Position: 1, - }, - &analysis.Token{ - Term: []byte("shingles"), - Type: analysis.Shingle, - Position: 5, - }, - &analysis.Token{ - Term: []byte("see shingles"), - Type: analysis.Shingle, - Position: 4, - }, - &analysis.Token{ - Term: []byte("_ see shingles"), - Type: analysis.Shingle, - Position: 4, - }, - &analysis.Token{ - Term: []byte("text _ see shingles"), - Type: analysis.Shingle, - Position: 2, - }, - &analysis.Token{ - Term: []byte("test text _ see shingles"), - Type: analysis.Shingle, - Position: 1, - }, - }, - }, - { - min: 2, - max: 2, - outputOriginal: true, - separator: " ", - filler: "_", - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("quick"), - }, - &analysis.Token{ - Term: []byte("brown"), - }, - &analysis.Token{ - Term: []byte("fox"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("quick"), - }, - &analysis.Token{ - Term: []byte("the quick"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("brown"), - }, - &analysis.Token{ - Term: []byte("quick brown"), - Type: analysis.Shingle, - }, - &analysis.Token{ - Term: []byte("fox"), - }, - &analysis.Token{ - Term: []byte("brown fox"), - Type: analysis.Shingle, - }, - }, - }, - } - - for _, test := range tests { - shingleFilter := NewShingleFilter(test.min, test.max, test.outputOriginal, test.separator, test.filler) - actual := shingleFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/README.md b/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/README.md deleted file mode 100644 index 56b0e863..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/README.md +++ /dev/null @@ -1,18 +0,0 @@ -## Languages supported - -"danish", -"dutch", -"english", -"finnish", -"french", -"german", -"hungarian", -"italian", -"norwegian", -"porter", -"portuguese", -"romanian", -"russian", -"spanish", -"swedish", -"turkish" \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter.go deleted file mode 100644 index 44a6e677..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package stemmer_filter - -import ( - "fmt" - - "bitbucket.org/tebeka/snowball" - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "stem" - -type StemmerFilter struct { - lang string - stemmerPool chan *snowball.Stemmer -} - -func NewStemmerFilter(lang string) (*StemmerFilter, error) { - stemmerPool := make(chan *snowball.Stemmer, 4) - for i := 0; i < 4; i++ { - stemmer, err := snowball.New(lang) - if err != nil { - return nil, err - } - stemmerPool <- stemmer - } - return &StemmerFilter{ - lang: lang, - stemmerPool: stemmerPool, - }, nil -} - -func MustNewStemmerFilter(lang string) *StemmerFilter { - sf, err := NewStemmerFilter(lang) - if err != nil { - panic(err) - } - return sf -} - -func (s *StemmerFilter) List() []string { - return snowball.LangList() -} - -func (s *StemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - // if it is not a protected keyword, stem it - if !token.KeyWord { - stemmer := <-s.stemmerPool - stemmed := stemmer.Stem(string(token.Term)) - s.stemmerPool <- stemmer - token.Term = []byte(stemmed) - } - } - return input -} - -func StemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - langVal, ok := config["lang"].(string) - if !ok { - return nil, fmt.Errorf("must specify stemmer language") - } - lang := langVal - return NewStemmerFilter(lang) -} - -func init() { - registry.RegisterTokenFilter(Name, StemmerFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter_test.go deleted file mode 100644 index 61a7c98f..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter/stemmer_filter_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package stemmer_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestStemmerFilter(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("walking"), - }, - &analysis.Token{ - Term: []byte("talked"), - }, - &analysis.Token{ - Term: []byte("business"), - }, - &analysis.Token{ - Term: []byte("protected"), - KeyWord: true, - }, - } - - expectedTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("walk"), - }, - &analysis.Token{ - Term: []byte("talk"), - }, - &analysis.Token{ - Term: []byte("busi"), - }, - &analysis.Token{ - Term: []byte("protected"), - KeyWord: true, - }, - } - - filter, err := NewStemmerFilter("en") - if err != nil { - t.Fatal(err) - } - ouputTokenStream := filter.Filter(inputTokenStream) - if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { - t.Errorf("expected %#v got %#v", expectedTokenStream[3], ouputTokenStream[3]) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter/stop_tokens_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter/stop_tokens_filter_test.go deleted file mode 100644 index 4ed666c1..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter/stop_tokens_filter_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package stop_tokens_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/token_map" - "github.com/blevesearch/bleve/registry" -) - -func TestStopWordsFilter(t *testing.T) { - - inputTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("a"), - }, - &analysis.Token{ - Term: []byte("walk"), - }, - &analysis.Token{ - Term: []byte("in"), - }, - &analysis.Token{ - Term: []byte("the"), - }, - &analysis.Token{ - Term: []byte("park"), - }, - } - - expectedTokenStream := analysis.TokenStream{ - &analysis.Token{ - Term: []byte("walk"), - }, - &analysis.Token{ - Term: []byte("park"), - }, - } - - cache := registry.NewCache() - stopListConfig := map[string]interface{}{ - "type": token_map.Name, - "tokens": []interface{}{"a", "in", "the"}, - } - _, err := cache.DefineTokenMap("stop_test", stopListConfig) - if err != nil { - t.Fatal(err) - } - - stopConfig := map[string]interface{}{ - "type": "stop_tokens", - "stop_token_map": "stop_test", - } - stopFilter, err := cache.DefineTokenFilter("stop_test", stopConfig) - if err != nil { - t.Fatal(err) - } - - ouputTokenStream := stopFilter.Filter(inputTokenStream) - if !reflect.DeepEqual(ouputTokenStream, expectedTokenStream) { - t.Errorf("expected %#v got %#v", expectedTokenStream, ouputTokenStream) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter.go deleted file mode 100644 index 2b16cb08..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package truncate_token_filter - -import ( - "bytes" - "fmt" - "unicode/utf8" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "truncate_token" - -type TruncateTokenFilter struct { - length int -} - -func NewTruncateTokenFilter(length int) *TruncateTokenFilter { - return &TruncateTokenFilter{ - length: length, - } -} - -func (s *TruncateTokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - wordLen := utf8.RuneCount(token.Term) - if wordLen > s.length { - runes := bytes.Runes(token.Term)[0:s.length] - newterm := make([]byte, 0, s.length*4) - for _, r := range runes { - runeBytes := make([]byte, utf8.RuneLen(r)) - utf8.EncodeRune(runeBytes, r) - newterm = append(newterm, runeBytes...) - } - token.Term = newterm - } - } - return input -} - -func TruncateTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - lenVal, ok := config["length"].(float64) - if !ok { - return nil, fmt.Errorf("must specify length") - } - length := int(lenVal) - - return NewTruncateTokenFilter(length), nil -} - -func init() { - registry.RegisterTokenFilter(Name, TruncateTokenFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter_test.go deleted file mode 100644 index cfe634ef..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter/truncate_token_filter_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package truncate_token_filter - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestTruncateTokenFilter(t *testing.T) { - - tests := []struct { - length int - input analysis.TokenStream - output analysis.TokenStream - }{ - { - length: 5, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcdefgh"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("abcde"), - }, - }, - }, - { - length: 3, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こんにちは世界"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("こんに"), - }, - }, - }, - { - length: 10, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("แยกคำภาษาไ"), - }, - }, - }, - } - - for _, test := range tests { - truncateTokenFilter := NewTruncateTokenFilter(test.length) - actual := truncateTokenFilter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize.go deleted file mode 100644 index dac6171c..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package unicode_normalize - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" - "golang.org/x/text/unicode/norm" -) - -const Name = "normalize_unicode" - -const NFC = "nfc" -const NFD = "nfd" -const NFKC = "nfkc" -const NFKD = "nfkd" - -var forms = map[string]norm.Form{ - NFC: norm.NFC, - NFD: norm.NFD, - NFKC: norm.NFKC, - NFKD: norm.NFKD, -} - -type UnicodeNormalizeFilter struct { - form norm.Form -} - -func NewUnicodeNormalizeFilter(formName string) (*UnicodeNormalizeFilter, error) { - form, ok := forms[formName] - if !ok { - return nil, fmt.Errorf("no form named %s", formName) - } - return &UnicodeNormalizeFilter{ - form: form, - }, nil -} - -func MustNewUnicodeNormalizeFilter(formName string) *UnicodeNormalizeFilter { - filter, err := NewUnicodeNormalizeFilter(formName) - if err != nil { - panic(err) - } - return filter -} - -func (s *UnicodeNormalizeFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - for _, token := range input { - token.Term = s.form.Bytes(token.Term) - } - return input -} - -func UnicodeNormalizeFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - formVal, ok := config["form"].(string) - if !ok { - return nil, fmt.Errorf("must specify form") - } - form := formVal - return NewUnicodeNormalizeFilter(form) -} - -func init() { - registry.RegisterTokenFilter(Name, UnicodeNormalizeFilterConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize_test.go deleted file mode 100644 index b415c652..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize/unicode_normalize_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package unicode_normalize - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -// the following tests come from the lucene -// test cases for CJK width filter -// which is our basis for using this -// as a substitute for that -func TestUnicodeNormalization(t *testing.T) { - - tests := []struct { - formName string - input analysis.TokenStream - output analysis.TokenStream - }{ - { - formName: NFKD, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Ｔｅｓｔ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("Test"), - }, - }, - }, - { - formName: NFKD, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("１２３４"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("1234"), - }, - }, - }, - { - formName: NFKD, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ｶﾀｶﾅ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("カタカナ"), - }, - }, - }, - { - formName: NFKC, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ｳﾞｨｯﾂ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ヴィッツ"), - }, - }, - }, - { - formName: NFKC, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("ﾊﾟﾅｿﾆｯｸ"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("パナソニック"), - }, - }, - }, - { - formName: NFD, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u212B"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0041\u030A"), - }, - }, - }, - { - formName: NFC, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u212B"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u00C5"), - }, - }, - }, - { - formName: NFKD, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\uFB01"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0066\u0069"), - }, - }, - }, - { - formName: NFKC, - input: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\uFB01"), - }, - }, - output: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("\u0066\u0069"), - }, - }, - }, - } - - for _, test := range tests { - filter := MustNewUnicodeNormalizeFilter(test.formName) - actual := filter.Filter(test.input) - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term) - t.Errorf("expected %#v, got %#v", test.output[0].Term, actual[0].Term) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_map/custom.go b/vendor/github.com/blevesearch/bleve/analysis/token_map/custom.go deleted file mode 100644 index eb5ccb68..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_map/custom.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package token_map - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "custom" - -func GenericTokenMapConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenMap, error) { - rv := analysis.NewTokenMap() - - // first: try to load by filename - filename, ok := config["filename"].(string) - if ok { - err := rv.LoadFile(filename) - return rv, err - } - // next: look for an inline word list - tokens, ok := config["tokens"].([]interface{}) - if ok { - for _, token := range tokens { - tokenStr, ok := token.(string) - if ok { - rv.AddToken(tokenStr) - } - } - return rv, nil - } - return nil, fmt.Errorf("must specify filename or list of tokens for token map") -} - -func init() { - registry.RegisterTokenMap(Name, GenericTokenMapConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_map_test.go b/vendor/github.com/blevesearch/bleve/analysis/token_map_test.go deleted file mode 100644 index c853b3c4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token_map_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package analysis - -import ( - "reflect" - "testing" -) - -func TestTokenMapLoadFile(t *testing.T) { - tokenMap := NewTokenMap() - err := tokenMap.LoadFile("test_words.txt") - if err != nil { - t.Fatal(err) - } - - expectedTokens := NewTokenMap() - expectedTokens.AddToken("marty") - expectedTokens.AddToken("steve") - expectedTokens.AddToken("dustin") - expectedTokens.AddToken("siri") - expectedTokens.AddToken("multiple") - expectedTokens.AddToken("words") - expectedTokens.AddToken("with") - expectedTokens.AddToken("different") - expectedTokens.AddToken("whitespace") - - if !reflect.DeepEqual(tokenMap, expectedTokens) { - t.Errorf("expected %#v, got %#v", expectedTokens, tokenMap) - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go new file mode 100644 index 00000000..39e38b45 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/analysis/tokenizer/unicode/unicode.go @@ -0,0 +1,131 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package unicode + +import ( + "github.com/blevesearch/segment" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/registry" +) + +const Name = "unicode" + +type UnicodeTokenizer struct { +} + +func NewUnicodeTokenizer() *UnicodeTokenizer { + return &UnicodeTokenizer{} +} + +func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream { + rvx := make([]analysis.TokenStream, 0, 10) // When rv gets full, append to rvx. + rv := make(analysis.TokenStream, 0, 1) + + ta := []analysis.Token(nil) + taNext := 0 + + segmenter := segment.NewWordSegmenterDirect(input) + start := 0 + pos := 1 + + guessRemaining := func(end int) int { + avgSegmentLen := end / (len(rv) + 1) + if avgSegmentLen < 1 { + avgSegmentLen = 1 + } + + remainingLen := len(input) - end + + return remainingLen / avgSegmentLen + } + + for segmenter.Segment() { + segmentBytes := segmenter.Bytes() + end := start + len(segmentBytes) + if segmenter.Type() != segment.None { + if taNext >= len(ta) { + remainingSegments := guessRemaining(end) + if remainingSegments > 1000 { + remainingSegments = 1000 + } + if remainingSegments < 1 { + remainingSegments = 1 + } + + ta = make([]analysis.Token, remainingSegments) + taNext = 0 + } + + token := &ta[taNext] + taNext++ + + token.Term = segmentBytes + token.Start = start + token.End = end + token.Position = pos + token.Type = convertType(segmenter.Type()) + + if len(rv) >= cap(rv) { // When rv is full, save it into rvx. + rvx = append(rvx, rv) + + rvCap := cap(rv) * 2 + if rvCap > 256 { + rvCap = 256 + } + + rv = make(analysis.TokenStream, 0, rvCap) // Next rv cap is bigger. + } + + rv = append(rv, token) + pos++ + } + start = end + } + + if len(rvx) > 0 { + n := len(rv) + for _, r := range rvx { + n += len(r) + } + rall := make(analysis.TokenStream, 0, n) + for _, r := range rvx { + rall = append(rall, r...) + } + return append(rall, rv...) + } + + return rv +} + +func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { + return NewUnicodeTokenizer(), nil +} + +func init() { + registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor) +} + +func convertType(segmentWordType int) analysis.TokenType { + switch segmentWordType { + case segment.Ideo: + return analysis.Ideographic + case segment.Kana: + return analysis.Ideographic + case segment.Number: + return analysis.Numeric + } + return analysis.AlphaNumeric +} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception.go deleted file mode 100644 index 0000f4c6..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package exception - -import ( - "fmt" - "regexp" - "strings" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "exception" - -type ExceptionsTokenizer struct { - exception *regexp.Regexp - remaining analysis.Tokenizer -} - -func NewExceptionsTokenizer(exception *regexp.Regexp, remaining analysis.Tokenizer) *ExceptionsTokenizer { - return &ExceptionsTokenizer{ - exception: exception, - remaining: remaining, - } -} - -func (t *ExceptionsTokenizer) Tokenize(input []byte) analysis.TokenStream { - rv := make(analysis.TokenStream, 0) - matches := t.exception.FindAllIndex(input, -1) - currInput := 0 - lastPos := 0 - for _, match := range matches { - start := match[0] - end := match[1] - if start > currInput { - // need to defer to remaining for unprocessed section - intermediate := t.remaining.Tokenize(input[currInput:start]) - // add intermediate tokens to our result stream - for _, token := range intermediate { - // adjust token offsets - token.Position += lastPos - token.Start += currInput - token.End += currInput - rv = append(rv, token) - } - lastPos += len(intermediate) - currInput = start - } - - // create single token with this regexp match - token := &analysis.Token{ - Term: input[start:end], - Start: start, - End: end, - Position: lastPos + 1, - } - rv = append(rv, token) - lastPos++ - currInput = end - - } - - if currInput < len(input) { - // need to defer to remaining for unprocessed section - intermediate := t.remaining.Tokenize(input[currInput:]) - // add intermediate tokens to our result stream - for _, token := range intermediate { - // adjust token offsets - token.Position += lastPos - token.Start += currInput - token.End += currInput - rv = append(rv, token) - } - } - - return rv -} - -func ExceptionsTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - exceptions := []string{} - iexceptions, ok := config["exceptions"].([]interface{}) - if ok { - for _, exception := range iexceptions { - exception, ok := exception.(string) - if ok { - exceptions = append(exceptions, exception) - } - } - } - aexceptions, ok := config["exceptions"].([]string) - if ok { - exceptions = append(exceptions, aexceptions...) - } - exceptionPattern := strings.Join(exceptions, "|") - r, err := regexp.Compile(exceptionPattern) - if err != nil { - return nil, fmt.Errorf("unable to build regexp tokenizer: %v", err) - } - - remainingName, ok := config["tokenizer"].(string) - if !ok { - return nil, fmt.Errorf("must specify tokenizer for remaining input") - } - remaining, err := cache.TokenizerNamed(remainingName) - if err != nil { - return nil, err - } - return NewExceptionsTokenizer(r, remaining), nil -} - -func init() { - registry.RegisterTokenizer(Name, ExceptionsTokenizerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception_test.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception_test.go deleted file mode 100644 index 62648f64..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/exception/exception_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package exception - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - _ "github.com/blevesearch/bleve/analysis/tokenizers/unicode" - "github.com/blevesearch/bleve/registry" -) - -func TestExceptionsTokenizer(t *testing.T) { - tests := []struct { - config map[string]interface{} - input []byte - patterns []string - result analysis.TokenStream - }{ - { - input: []byte("test http://blevesearch.com/ words"), - config: map[string]interface{}{ - "type": "exception", - "tokenizer": "unicode", - "exceptions": []interface{}{ - `[hH][tT][tT][pP][sS]?://(\S)*`, - `[fF][iI][lL][eE]://(\S)*`, - `[fF][tT][pP]://(\S)*`, - }, - }, - result: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("test"), - Position: 1, - Start: 0, - End: 4, - }, - &analysis.Token{ - Term: []byte("http://blevesearch.com/"), - Position: 2, - Start: 5, - End: 28, - }, - &analysis.Token{ - Term: []byte("words"), - Position: 3, - Start: 29, - End: 34, - }, - }, - }, - { - input: []byte("what ftp://blevesearch.com/ songs"), - config: map[string]interface{}{ - "type": "exception", - "tokenizer": "unicode", - "exceptions": []interface{}{ - `[hH][tT][tT][pP][sS]?://(\S)*`, - `[fF][iI][lL][eE]://(\S)*`, - `[fF][tT][pP]://(\S)*`, - }, - }, - result: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("what"), - Position: 1, - Start: 0, - End: 4, - }, - &analysis.Token{ - Term: []byte("ftp://blevesearch.com/"), - Position: 2, - Start: 5, - End: 27, - }, - &analysis.Token{ - Term: []byte("songs"), - Position: 3, - Start: 28, - End: 33, - }, - }, - }, - { - input: []byte("please email marty@couchbase.com the URL https://blevesearch.com/"), - config: map[string]interface{}{ - "type": "exception", - "tokenizer": "unicode", - "exceptions": []interface{}{ - `[hH][tT][tT][pP][sS]?://(\S)*`, - `[fF][iI][lL][eE]://(\S)*`, - `[fF][tT][pP]://(\S)*`, - `\S+@\S+`, - }, - }, - result: analysis.TokenStream{ - &analysis.Token{ - Term: []byte("please"), - Position: 1, - Start: 0, - End: 6, - }, - &analysis.Token{ - Term: []byte("email"), - Position: 2, - Start: 7, - End: 12, - }, - &analysis.Token{ - Term: []byte("marty@couchbase.com"), - Position: 3, - Start: 13, - End: 32, - }, - &analysis.Token{ - Term: []byte("the"), - Position: 4, - Start: 33, - End: 36, - }, - &analysis.Token{ - Term: []byte("URL"), - Position: 5, - Start: 37, - End: 40, - }, - &analysis.Token{ - Term: []byte("https://blevesearch.com/"), - Position: 6, - Start: 41, - End: 65, - }, - }, - }, - } - - // remaining := unicode.NewUnicodeTokenizer() - for _, test := range tests { - - // build the requested exception tokenizer - cache := registry.NewCache() - tokenizer, err := cache.DefineTokenizer("custom", test.config) - if err != nil { - t.Fatal(err) - } - - // pattern := strings.Join(test.patterns, "|") - // r, err := regexp.Compile(pattern) - // if err != nil { - // t.Fatal(err) - // } - // tokenizer := NewExceptionsTokenizer(r, remaining) - actual := tokenizer.Tokenize(test.input) - if !reflect.DeepEqual(actual, test.result) { - t.Errorf("expected %v, got %v", test.result, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary.go deleted file mode 100644 index 8e6c5dd4..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package icu - -// #cgo LDFLAGS: -licuuc -licudata -// #include -// #include -// #include "unicode/utypes.h" -// #include "unicode/uchar.h" -// #include "unicode/ubrk.h" -// #include "unicode/ustring.h" -import "C" - -import ( - "unsafe" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "icu" - -type UnicodeWordBoundaryTokenizer struct { - locale *C.char -} - -func NewUnicodeWordBoundaryTokenizer() *UnicodeWordBoundaryTokenizer { - return &UnicodeWordBoundaryTokenizer{} -} - -func NewUnicodeWordBoundaryCustomLocaleTokenizer(locale string) *UnicodeWordBoundaryTokenizer { - return &UnicodeWordBoundaryTokenizer{ - locale: C.CString(locale), - } -} - -func (t *UnicodeWordBoundaryTokenizer) Tokenize(input []byte) analysis.TokenStream { - rv := make(analysis.TokenStream, 0) - - if len(input) < 1 { - return rv - } - - // works - var myUnsafePointer = unsafe.Pointer(&(input[0])) - var myCCharPointer *C.char = (*C.char)(myUnsafePointer) - - var inlen C.int32_t = C.int32_t(len(input)) - var buflen C.int32_t = C.int32_t(2*len(input) + 1) // worse case each byte becomes 2 - var stringToExamine []C.UChar = make([]C.UChar, buflen) - var myUnsafePointerToExamine = unsafe.Pointer(&(stringToExamine[0])) - var myUCharPointer *C.UChar = (*C.UChar)(myUnsafePointerToExamine) - C.u_uastrncpy(myUCharPointer, myCCharPointer, inlen) - - var err C.UErrorCode = C.U_ZERO_ERROR - bi := C.ubrk_open(C.UBRK_WORD, t.locale, myUCharPointer, -1, &err) - - if err > C.U_ZERO_ERROR { - return rv - } - - defer C.ubrk_close(bi) - - position := 0 - var prev C.int32_t - p := C.ubrk_first(bi) - for p != C.UBRK_DONE { - - q := C.ubrk_getRuleStatus(bi) - - // convert boundaries back to utf8 positions - var nilCString *C.char - var indexA C.int32_t - - C.u_strToUTF8(nilCString, 0, &indexA, myUCharPointer, prev, &err) - if err > C.U_ZERO_ERROR && err != C.U_BUFFER_OVERFLOW_ERROR { - return rv - } else { - err = C.U_ZERO_ERROR - } - - var indexB C.int32_t - C.u_strToUTF8(nilCString, 0, &indexB, myUCharPointer, p, &err) - if err > C.U_ZERO_ERROR && err != C.U_BUFFER_OVERFLOW_ERROR { - return rv - } else { - err = C.U_ZERO_ERROR - } - - if q != 0 { - position += 1 - token := analysis.Token{ - Start: int(indexA), - End: int(indexB), - Term: input[indexA:indexB], - Position: position, - Type: analysis.AlphaNumeric, - } - if q == 100 { - token.Type = analysis.Numeric - } - if q == 400 { - token.Type = analysis.Ideographic - } - rv = append(rv, &token) - } - prev = p - p = C.ubrk_next(bi) - } - - return rv -} - -func UnicodeWordBoundaryTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - locale := "" - localeVal, ok := config["locale"].(string) - if ok { - locale = localeVal - } - if locale == "" { - return NewUnicodeWordBoundaryTokenizer(), nil - } else { - return NewUnicodeWordBoundaryCustomLocaleTokenizer(locale), nil - } -} - -func init() { - registry.RegisterTokenizer(Name, UnicodeWordBoundaryTokenizerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary_test.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary_test.go deleted file mode 100644 index 65c0db52..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/icu/boundary_test.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package icu - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestBoundary(t *testing.T) { - - tests := []struct { - input []byte - locale string - output analysis.TokenStream - }{ - { - []byte("Hello World"), - "en_US", - analysis.TokenStream{ - { - Start: 0, - End: 5, - Term: []byte("Hello"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 6, - End: 11, - Term: []byte("World"), - Position: 2, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("steven's"), - "en_US", - analysis.TokenStream{ - { - Start: 0, - End: 8, - Term: []byte("steven's"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("こんにちは世界"), - "en_US", - analysis.TokenStream{ - { - Start: 0, - End: 15, - Term: []byte("こんにちは"), - Position: 1, - Type: analysis.Ideographic, - }, - { - Start: 15, - End: 21, - Term: []byte("世界"), - Position: 2, - Type: analysis.Ideographic, - }, - }, - }, - { - []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), - "th_TH", - analysis.TokenStream{ - { - Start: 0, - End: 9, - Term: []byte("แยก"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 9, - End: 15, - Term: []byte("คำ"), - Position: 2, - Type: analysis.AlphaNumeric, - }, - { - Start: 15, - End: 27, - Term: []byte("ภาษา"), - Position: 3, - Type: analysis.AlphaNumeric, - }, - { - Start: 27, - End: 36, - Term: []byte("ไทย"), - Position: 4, - Type: analysis.AlphaNumeric, - }, - { - Start: 36, - End: 42, - Term: []byte("ก็"), - Position: 5, - Type: analysis.AlphaNumeric, - }, - { - Start: 42, - End: 57, - Term: []byte("ทำได้"), - Position: 6, - Type: analysis.AlphaNumeric, - }, - { - Start: 57, - End: 63, - Term: []byte("นะ"), - Position: 7, - Type: analysis.AlphaNumeric, - }, - { - Start: 63, - End: 72, - Term: []byte("จ้ะ"), - Position: 8, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("age 25"), - "en_US", - analysis.TokenStream{ - { - Start: 0, - End: 3, - Term: []byte("age"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 4, - End: 6, - Term: []byte("25"), - Position: 2, - Type: analysis.Numeric, - }, - }, - }, - } - - for _, test := range tests { - tokenizer := NewUnicodeWordBoundaryCustomLocaleTokenizer(test.locale) - actual := tokenizer.Tokenize(test.input) - - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) - } - } -} - -var sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE: -If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). -The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. -When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. -Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. -If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`) - -func BenchmarkTokenizeEnglishText(b *testing.B) { - - tokenizer := NewUnicodeWordBoundaryCustomLocaleTokenizer("en_US") - b.ResetTimer() - - for i := 0; i < b.N; i++ { - tokenizer.Tokenize(sampleLargeInput) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer.go deleted file mode 100644 index 20e04cd5..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package regexp_tokenizer - -import ( - "fmt" - "regexp" - "strconv" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "regexp" - -var IdeographRegexp = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`) - -type RegexpTokenizer struct { - r *regexp.Regexp -} - -func NewRegexpTokenizer(r *regexp.Regexp) *RegexpTokenizer { - return &RegexpTokenizer{ - r: r, - } -} - -func (rt *RegexpTokenizer) Tokenize(input []byte) analysis.TokenStream { - matches := rt.r.FindAllIndex(input, -1) - rv := make(analysis.TokenStream, len(matches)) - for i, match := range matches { - matchBytes := input[match[0]:match[1]] - token := analysis.Token{ - Term: matchBytes, - Start: match[0], - End: match[1], - Position: i + 1, - Type: detectTokenType(matchBytes), - } - rv[i] = &token - } - return rv -} - -func RegexpTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - rval, ok := config["regexp"].(string) - if !ok { - return nil, fmt.Errorf("must specify regexp") - } - r, err := regexp.Compile(rval) - if err != nil { - return nil, fmt.Errorf("unable to build regexp tokenizer: %v", err) - } - return NewRegexpTokenizer(r), nil -} - -func init() { - registry.RegisterTokenizer(Name, RegexpTokenizerConstructor) -} - -func detectTokenType(termBytes []byte) analysis.TokenType { - if IdeographRegexp.Match(termBytes) { - return analysis.Ideographic - } - _, err := strconv.ParseFloat(string(termBytes), 64) - if err == nil { - return analysis.Numeric - } - return analysis.AlphaNumeric -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go deleted file mode 100644 index d1d119f6..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer/regexp_tokenizer_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package regexp_tokenizer - -import ( - "reflect" - "regexp" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestBoundary(t *testing.T) { - - wordRegex := regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}|\w+`) - - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - []byte("Hello World."), - analysis.TokenStream{ - { - Start: 0, - End: 5, - Term: []byte("Hello"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 6, - End: 11, - Term: []byte("World"), - Position: 2, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("こんにちは世界"), - analysis.TokenStream{ - { - Start: 0, - End: 3, - Term: []byte("こ"), - Position: 1, - Type: analysis.Ideographic, - }, - { - Start: 3, - End: 6, - Term: []byte("ん"), - Position: 2, - Type: analysis.Ideographic, - }, - { - Start: 6, - End: 9, - Term: []byte("に"), - Position: 3, - Type: analysis.Ideographic, - }, - { - Start: 9, - End: 12, - Term: []byte("ち"), - Position: 4, - Type: analysis.Ideographic, - }, - { - Start: 12, - End: 15, - Term: []byte("は"), - Position: 5, - Type: analysis.Ideographic, - }, - { - Start: 15, - End: 18, - Term: []byte("世"), - Position: 6, - Type: analysis.Ideographic, - }, - { - Start: 18, - End: 21, - Term: []byte("界"), - Position: 7, - Type: analysis.Ideographic, - }, - }, - }, - { - []byte(""), - analysis.TokenStream{}, - }, - } - - for _, test := range tests { - tokenizer := NewRegexpTokenizer(wordRegex) - actual := tokenizer.Tokenize(test.input) - - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token.go deleted file mode 100644 index eaef7baa..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package single_token - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "single" - -type SingleTokenTokenizer struct { -} - -func NewSingleTokenTokenizer() *SingleTokenTokenizer { - return &SingleTokenTokenizer{} -} - -func (t *SingleTokenTokenizer) Tokenize(input []byte) analysis.TokenStream { - return analysis.TokenStream{ - &analysis.Token{ - Term: input, - Position: 1, - Start: 0, - End: len(input), - Type: analysis.AlphaNumeric, - }, - } -} - -func SingleTokenTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - return NewSingleTokenTokenizer(), nil -} - -func init() { - registry.RegisterTokenizer(Name, SingleTokenTokenizerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token_test.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token_test.go deleted file mode 100644 index 1bd50cda..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/single_token/single_token_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package single_token - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" -) - -func TestSingleTokenTokenizer(t *testing.T) { - - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - []byte("Hello World"), - analysis.TokenStream{ - { - Start: 0, - End: 11, - Term: []byte("Hello World"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("こんにちは世界"), - analysis.TokenStream{ - { - Start: 0, - End: 21, - Term: []byte("こんにちは世界"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), - analysis.TokenStream{ - { - Start: 0, - End: 72, - Term: []byte("แยกคำภาษาไทยก็ทำได้นะจ้ะ"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - } - - for _, test := range tests { - tokenizer := NewSingleTokenTokenizer() - actual := tokenizer.Tokenize(test.input) - - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode.go deleted file mode 100644 index b957231d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package unicode - -import ( - "github.com/blevesearch/segment" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "unicode" - -type UnicodeTokenizer struct { -} - -func NewUnicodeTokenizer() *UnicodeTokenizer { - return &UnicodeTokenizer{} -} - -func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream { - - rv := make(analysis.TokenStream, 0) - - segmenter := segment.NewWordSegmenterDirect(input) - start := 0 - pos := 1 - for segmenter.Segment() { - segmentBytes := segmenter.Bytes() - end := start + len(segmentBytes) - if segmenter.Type() != segment.None { - token := analysis.Token{ - Term: segmentBytes, - Start: start, - End: end, - Position: pos, - Type: convertType(segmenter.Type()), - } - rv = append(rv, &token) - pos++ - } - start = end - - } - return rv -} - -func UnicodeTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - return NewUnicodeTokenizer(), nil -} - -func init() { - registry.RegisterTokenizer(Name, UnicodeTokenizerConstructor) -} - -func convertType(segmentWordType int) analysis.TokenType { - switch segmentWordType { - case segment.Ideo: - return analysis.Ideographic - case segment.Kana: - return analysis.Ideographic - case segment.Number: - return analysis.Numeric - } - return analysis.AlphaNumeric -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode_test.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode_test.go deleted file mode 100644 index c3037cd3..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/unicode/unicode_test.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package unicode - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/segment" -) - -func TestUnicode(t *testing.T) { - - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - []byte("Hello World"), - analysis.TokenStream{ - { - Start: 0, - End: 5, - Term: []byte("Hello"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 6, - End: 11, - Term: []byte("World"), - Position: 2, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("steven's"), - analysis.TokenStream{ - { - Start: 0, - End: 8, - Term: []byte("steven's"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("こんにちは世界"), - analysis.TokenStream{ - { - Start: 0, - End: 3, - Term: []byte("こ"), - Position: 1, - Type: analysis.Ideographic, - }, - { - Start: 3, - End: 6, - Term: []byte("ん"), - Position: 2, - Type: analysis.Ideographic, - }, - { - Start: 6, - End: 9, - Term: []byte("に"), - Position: 3, - Type: analysis.Ideographic, - }, - { - Start: 9, - End: 12, - Term: []byte("ち"), - Position: 4, - Type: analysis.Ideographic, - }, - { - Start: 12, - End: 15, - Term: []byte("は"), - Position: 5, - Type: analysis.Ideographic, - }, - { - Start: 15, - End: 18, - Term: []byte("世"), - Position: 6, - Type: analysis.Ideographic, - }, - { - Start: 18, - End: 21, - Term: []byte("界"), - Position: 7, - Type: analysis.Ideographic, - }, - }, - }, - { - []byte("age 25"), - analysis.TokenStream{ - { - Start: 0, - End: 3, - Term: []byte("age"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 4, - End: 6, - Term: []byte("25"), - Position: 2, - Type: analysis.Numeric, - }, - }, - }, - { - []byte("カ"), - analysis.TokenStream{ - { - Start: 0, - End: 3, - Term: []byte("カ"), - Position: 1, - Type: analysis.Ideographic, - }, - }, - }, - } - - for _, test := range tests { - tokenizer := NewUnicodeTokenizer() - actual := tokenizer.Tokenize(test.input) - - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) - } - } -} - -var sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE: -If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). -The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. -When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. -Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. -If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`) - -func BenchmarkTokenizeEnglishText(b *testing.B) { - - tokenizer := NewUnicodeTokenizer() - b.ResetTimer() - - for i := 0; i < b.N; i++ { - tokenizer.Tokenize(sampleLargeInput) - } - -} - -func TestConvertType(t *testing.T) { - tests := []struct { - in int - out analysis.TokenType - }{ - { - segment.Ideo, analysis.Ideographic, - }, - { - segment.Kana, analysis.Ideographic, - }, - { - segment.Number, analysis.Numeric, - }, - { - segment.Letter, analysis.AlphaNumeric, - }, - } - - for _, test := range tests { - actual := convertType(test.in) - if actual != test.out { - t.Errorf("expected %d, got %d for %d", test.out, actual, test.in) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer.go deleted file mode 100644 index 870cdbd7..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package whitespace_tokenizer - -import ( - "regexp" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer" - "github.com/blevesearch/bleve/registry" -) - -const Name = "whitespace" - -var whitespaceTokenizerRegexp = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}|[^\p{Z}\p{P}\p{C}\p{Han}\p{Hangul}\p{Hiragana}\p{Katakana}]+`) - -func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) { - return regexp_tokenizer.NewRegexpTokenizer(whitespaceTokenizerRegexp), nil -} - -func init() { - registry.RegisterTokenizer(Name, TokenizerConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer_test.go b/vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer_test.go deleted file mode 100644 index 2ae1787d..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer/whitespace_tokenizer_test.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package whitespace_tokenizer - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer" -) - -func TestBoundary(t *testing.T) { - - tests := []struct { - input []byte - output analysis.TokenStream - }{ - { - []byte("Hello World."), - analysis.TokenStream{ - { - Start: 0, - End: 5, - Term: []byte("Hello"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 6, - End: 11, - Term: []byte("World"), - Position: 2, - Type: analysis.AlphaNumeric, - }, - }, - }, - { - []byte("こんにちは世界"), - analysis.TokenStream{ - { - Start: 0, - End: 3, - Term: []byte("こ"), - Position: 1, - Type: analysis.Ideographic, - }, - { - Start: 3, - End: 6, - Term: []byte("ん"), - Position: 2, - Type: analysis.Ideographic, - }, - { - Start: 6, - End: 9, - Term: []byte("に"), - Position: 3, - Type: analysis.Ideographic, - }, - { - Start: 9, - End: 12, - Term: []byte("ち"), - Position: 4, - Type: analysis.Ideographic, - }, - { - Start: 12, - End: 15, - Term: []byte("は"), - Position: 5, - Type: analysis.Ideographic, - }, - { - Start: 15, - End: 18, - Term: []byte("世"), - Position: 6, - Type: analysis.Ideographic, - }, - { - Start: 18, - End: 21, - Term: []byte("界"), - Position: 7, - Type: analysis.Ideographic, - }, - }, - }, - { - []byte(""), - analysis.TokenStream{}, - }, - { - []byte("abc界"), - analysis.TokenStream{ - { - Start: 0, - End: 3, - Term: []byte("abc"), - Position: 1, - Type: analysis.AlphaNumeric, - }, - { - Start: 3, - End: 6, - Term: []byte("界"), - Position: 2, - Type: analysis.Ideographic, - }, - }, - }, - } - - for _, test := range tests { - tokenizer := regexp_tokenizer.NewRegexpTokenizer(whitespaceTokenizerRegexp) - actual := tokenizer.Tokenize(test.input) - - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("Expected %v, got %v for %s", test.output, actual, string(test.input)) - } - } -} - -var sampleLargeInput = []byte(`There are three characteristics of liquids which are relevant to the discussion of a BLEVE: -If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). -The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. -When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. -Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, oxygen, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. -If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings.`) - -func BenchmarkTokenizeEnglishText(b *testing.B) { - - tokenizer := regexp_tokenizer.NewRegexpTokenizer(whitespaceTokenizerRegexp) - b.ResetTimer() - - for i := 0; i < b.N; i++ { - tokenizer.Tokenize(sampleLargeInput) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/analysis/token_map.go b/vendor/github.com/blevesearch/bleve/analysis/tokenmap.go similarity index 58% rename from vendor/github.com/blevesearch/bleve/analysis/token_map.go rename to vendor/github.com/blevesearch/bleve/analysis/tokenmap.go index 17a26f48..7c0d0a89 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/token_map.go +++ b/vendor/github.com/blevesearch/bleve/analysis/tokenmap.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package analysis @@ -23,6 +28,9 @@ func NewTokenMap() TokenMap { return make(TokenMap, 0) } +// LoadFile reads in a list of tokens from a text file, +// one per line. +// Comments are supported using `#` or `|` func (t TokenMap) LoadFile(filename string) error { data, err := ioutil.ReadFile(filename) if err != nil { @@ -31,6 +39,9 @@ func (t TokenMap) LoadFile(filename string) error { return t.LoadBytes(data) } +// LoadBytes reads in a list of tokens from memory, +// one per line. +// Comments are supported using `#` or `|` func (t TokenMap) LoadBytes(data []byte) error { bytesReader := bytes.NewReader(data) bufioReader := bufio.NewReader(bytesReader) diff --git a/vendor/github.com/blevesearch/bleve/analysis/type.go b/vendor/github.com/blevesearch/bleve/analysis/type.go index 0f32ece1..589cc1ca 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/type.go +++ b/vendor/github.com/blevesearch/bleve/analysis/type.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package analysis @@ -28,12 +33,22 @@ const ( Shingle Single Double + Boolean ) +// Token represents one occurrence of a term at a particular location in a +// field. type Token struct { - Start int `json:"start"` - End int `json:"end"` - Term []byte `json:"term"` + // Start specifies the byte offset of the beginning of the term in the + // field. + Start int `json:"start"` + + // End specifies the byte offset of the end of the term in the field. + End int `json:"end"` + Term []byte `json:"term"` + + // Position specifies the 1-based index of the token in the sequence of + // occurrences of its term in the field. Position int `json:"position"` Type TokenType `json:"type"` KeyWord bool `json:"keyword"` @@ -45,10 +60,13 @@ func (t *Token) String() string { type TokenStream []*Token +// A Tokenizer splits an input string into tokens, the usual behaviour being to +// map words to tokens. type Tokenizer interface { Tokenize([]byte) TokenStream } +// A TokenFilter adds, transforms or removes tokens from a token stream. type TokenFilter interface { Filter(TokenStream) TokenStream } diff --git a/vendor/github.com/blevesearch/bleve/analysis/util.go b/vendor/github.com/blevesearch/bleve/analysis/util.go index f15f08e4..8e4348a1 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/util.go +++ b/vendor/github.com/blevesearch/bleve/analysis/util.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package analysis @@ -34,14 +39,32 @@ func InsertRune(in []rune, pos int, r rune) []rune { return rv } -func BuildTermFromRunes(runes []rune) []byte { - rv := make([]byte, 0, len(runes)*4) +// BuildTermFromRunesOptimistic will build a term from the provided runes +// AND optimistically attempt to encode into the provided buffer +// if at any point it appears the buffer is too small, a new buffer is +// allocated and that is used instead +// this should be used in cases where frequently the new term is the same +// length or shorter than the original term (in number of bytes) +func BuildTermFromRunesOptimistic(buf []byte, runes []rune) []byte { + rv := buf + used := 0 for _, r := range runes { - runeBytes := make([]byte, utf8.RuneLen(r)) - utf8.EncodeRune(runeBytes, r) - rv = append(rv, runeBytes...) + nextLen := utf8.RuneLen(r) + if used+nextLen > len(rv) { + // alloc new buf + buf = make([]byte, len(runes)*utf8.UTFMax) + // copy work we've already done + copy(buf, rv[:used]) + rv = buf + } + written := utf8.EncodeRune(rv[used:], r) + used += written } - return rv + return rv[:used] +} + +func BuildTermFromRunes(runes []rune) []byte { + return BuildTermFromRunesOptimistic(make([]byte, len(runes)*utf8.UTFMax), runes) } func TruncateRunes(input []byte, num int) []byte { diff --git a/vendor/github.com/blevesearch/bleve/analysis/util_test.go b/vendor/github.com/blevesearch/bleve/analysis/util_test.go deleted file mode 100644 index b426abfa..00000000 --- a/vendor/github.com/blevesearch/bleve/analysis/util_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package analysis - -import ( - "reflect" - "testing" -) - -func TestDeleteRune(t *testing.T) { - tests := []struct { - in []rune - delPos int - out []rune - }{ - { - in: []rune{'a', 'b', 'c'}, - delPos: 1, - out: []rune{'a', 'c'}, - }, - } - - for _, test := range tests { - actual := DeleteRune(test.in, test.delPos) - if !reflect.DeepEqual(actual, test.out) { - t.Errorf("expected %#v, got %#v", test.out, actual) - } - } -} - -func TestInsertRune(t *testing.T) { - tests := []struct { - in []rune - insPos int - insRune rune - out []rune - }{ - { - in: []rune{'a', 'b', 'c'}, - insPos: 1, - insRune: 'x', - out: []rune{'a', 'x', 'b', 'c'}, - }, - { - in: []rune{'a', 'b', 'c'}, - insPos: 0, - insRune: 'x', - out: []rune{'x', 'a', 'b', 'c'}, - }, - { - in: []rune{'a', 'b', 'c'}, - insPos: 3, - insRune: 'x', - out: []rune{'a', 'b', 'c', 'x'}, - }, - } - - for _, test := range tests { - actual := InsertRune(test.in, test.insPos, test.insRune) - if !reflect.DeepEqual(actual, test.out) { - t.Errorf("expected %#v, got %#v", test.out, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/config.go b/vendor/github.com/blevesearch/bleve/config.go index 637be1e8..74d407fd 100644 --- a/vendor/github.com/blevesearch/bleve/config.go +++ b/vendor/github.com/blevesearch/bleve/config.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve @@ -15,101 +20,11 @@ import ( "log" "time" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/store/gtreap" + "github.com/blevesearch/bleve/index/upsidedown" "github.com/blevesearch/bleve/registry" - - // token maps - _ "github.com/blevesearch/bleve/analysis/token_map" - - // fragment formatters - _ "github.com/blevesearch/bleve/search/highlight/fragment_formatters/ansi" - _ "github.com/blevesearch/bleve/search/highlight/fragment_formatters/html" - - // fragmenters - _ "github.com/blevesearch/bleve/search/highlight/fragmenters/simple" - - // highlighters - _ "github.com/blevesearch/bleve/search/highlight/highlighters/simple" - - // char filters - _ "github.com/blevesearch/bleve/analysis/char_filters/html_char_filter" - _ "github.com/blevesearch/bleve/analysis/char_filters/regexp_char_filter" - _ "github.com/blevesearch/bleve/analysis/char_filters/zero_width_non_joiner" - - // analyzers - _ "github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer" - _ "github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer" - _ "github.com/blevesearch/bleve/analysis/analyzers/simple_analyzer" - _ "github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer" - - // token filters - _ "github.com/blevesearch/bleve/analysis/token_filters/apostrophe_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/compound" - _ "github.com/blevesearch/bleve/analysis/token_filters/edge_ngram_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/elision_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/keyword_marker_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/length_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/ngram_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/shingle" - _ "github.com/blevesearch/bleve/analysis/token_filters/stop_tokens_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/truncate_token_filter" - _ "github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize" - - // tokenizers - _ "github.com/blevesearch/bleve/analysis/tokenizers/exception" - _ "github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer" - _ "github.com/blevesearch/bleve/analysis/tokenizers/single_token" - _ "github.com/blevesearch/bleve/analysis/tokenizers/unicode" - _ "github.com/blevesearch/bleve/analysis/tokenizers/whitespace_tokenizer" - - // date time parsers - _ "github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional" - _ "github.com/blevesearch/bleve/analysis/datetime_parsers/flexible_go" - - // languages - _ "github.com/blevesearch/bleve/analysis/language/ar" - _ "github.com/blevesearch/bleve/analysis/language/bg" - _ "github.com/blevesearch/bleve/analysis/language/ca" - _ "github.com/blevesearch/bleve/analysis/language/cjk" - _ "github.com/blevesearch/bleve/analysis/language/ckb" - _ "github.com/blevesearch/bleve/analysis/language/cs" - _ "github.com/blevesearch/bleve/analysis/language/da" - _ "github.com/blevesearch/bleve/analysis/language/de" - _ "github.com/blevesearch/bleve/analysis/language/el" - _ "github.com/blevesearch/bleve/analysis/language/en" - _ "github.com/blevesearch/bleve/analysis/language/es" - _ "github.com/blevesearch/bleve/analysis/language/eu" - _ "github.com/blevesearch/bleve/analysis/language/fa" - _ "github.com/blevesearch/bleve/analysis/language/fi" - _ "github.com/blevesearch/bleve/analysis/language/fr" - _ "github.com/blevesearch/bleve/analysis/language/ga" - _ "github.com/blevesearch/bleve/analysis/language/gl" - _ "github.com/blevesearch/bleve/analysis/language/hi" - _ "github.com/blevesearch/bleve/analysis/language/hu" - _ "github.com/blevesearch/bleve/analysis/language/hy" - _ "github.com/blevesearch/bleve/analysis/language/id" - _ "github.com/blevesearch/bleve/analysis/language/in" - _ "github.com/blevesearch/bleve/analysis/language/it" - _ "github.com/blevesearch/bleve/analysis/language/nl" - _ "github.com/blevesearch/bleve/analysis/language/no" - _ "github.com/blevesearch/bleve/analysis/language/pt" - _ "github.com/blevesearch/bleve/analysis/language/ro" - _ "github.com/blevesearch/bleve/analysis/language/ru" - _ "github.com/blevesearch/bleve/analysis/language/sv" - _ "github.com/blevesearch/bleve/analysis/language/th" - _ "github.com/blevesearch/bleve/analysis/language/tr" - - // kv stores - _ "github.com/blevesearch/bleve/index/store/boltdb" - _ "github.com/blevesearch/bleve/index/store/goleveldb" - _ "github.com/blevesearch/bleve/index/store/gtreap" - _ "github.com/blevesearch/bleve/index/store/inmem" - - // byte array converters - _ "github.com/blevesearch/bleve/analysis/byte_array_converters/ignore" - _ "github.com/blevesearch/bleve/analysis/byte_array_converters/json" - _ "github.com/blevesearch/bleve/analysis/byte_array_converters/string" + "github.com/blevesearch/bleve/search/highlight/highlighter/html" ) var bleveExpVar = expvar.NewMap("bleve") @@ -118,14 +33,20 @@ type configuration struct { Cache *registry.Cache DefaultHighlighter string DefaultKVStore string + DefaultMemKVStore string + DefaultIndexType string SlowSearchLogThreshold time.Duration - analysisQueue *upside_down.AnalysisQueue + analysisQueue *index.AnalysisQueue +} + +func (c *configuration) SetAnalysisQueueSize(n int) { + c.analysisQueue = index.NewAnalysisQueue(n) } func newConfiguration() *configuration { return &configuration{ Cache: registry.NewCache(), - analysisQueue: upside_down.NewAnalysisQueue(4), + analysisQueue: index.NewAnalysisQueue(4), } } @@ -138,44 +59,24 @@ func init() { // build the default configuration Config = newConfiguration() - _, err := Config.Cache.DefineFragmentFormatter("highlightSpanHTML", - map[string]interface{}{ - "type": "html", - "before": ``, - "after": ``, - }) - if err != nil { - panic(err) - } - - _, err = Config.Cache.DefineHighlighter("html", - map[string]interface{}{ - "type": "simple", - "fragmenter": "simple", - "formatter": "highlightSpanHTML", - }) - if err != nil { - panic(err) - } - - _, err = Config.Cache.DefineHighlighter("ansi", - map[string]interface{}{ - "type": "simple", - "fragmenter": "simple", - "formatter": "ansi", - }) - if err != nil { - panic(err) - } - // set the default highlighter - Config.DefaultHighlighter = "html" + Config.DefaultHighlighter = html.Name // default kv store - Config.DefaultKVStore = "boltdb" + Config.DefaultKVStore = "" + + // default mem only kv store + Config.DefaultMemKVStore = gtreap.Name + + // default index + Config.DefaultIndexType = upsidedown.Name bootDuration := time.Since(bootStart) bleveExpVar.Add("bootDuration", int64(bootDuration)) + indexStats = NewIndexStats() + bleveExpVar.Set("indexes", indexStats) + + initDisk() } var logger = log.New(ioutil.Discard, "bleve", log.LstdFlags) diff --git a/vendor/github.com/blevesearch/bleve/config_app.go b/vendor/github.com/blevesearch/bleve/config_app.go new file mode 100644 index 00000000..112d0b60 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/config_app.go @@ -0,0 +1,23 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build appengine appenginevm + +package bleve + +// in the appengine environment we cannot support disk based indexes +// so we do no extra configuration in this method +func initDisk() { + +} diff --git a/vendor/github.com/blevesearch/bleve/config_cld2.go b/vendor/github.com/blevesearch/bleve/config_cld2.go deleted file mode 100644 index 84961cde..00000000 --- a/vendor/github.com/blevesearch/bleve/config_cld2.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build cld2 full - -package bleve - -import ( - // cld2 token filter - _ "github.com/blevesearch/bleve/analysis/token_filters/cld2" - - // detect language analyzer - _ "github.com/blevesearch/bleve/analysis/analyzers/detect_lang_analyzer" -) diff --git a/vendor/github.com/blevesearch/bleve/config_cznicb.go b/vendor/github.com/blevesearch/bleve/config_cznicb.go deleted file mode 100644 index 336ea79c..00000000 --- a/vendor/github.com/blevesearch/bleve/config_cznicb.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build go1.4 - -package bleve - -import ( - _ "github.com/blevesearch/bleve/index/store/cznicb" -) diff --git a/vendor/github.com/blevesearch/bleve/config_disk.go b/vendor/github.com/blevesearch/bleve/config_disk.go new file mode 100644 index 00000000..d03bceb4 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/config_disk.go @@ -0,0 +1,25 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !appengine,!appenginevm + +package bleve + +import "github.com/blevesearch/bleve/index/store/boltdb" + +// in normal environments we configure boltdb as the default storage +func initDisk() { + // default kv store + Config.DefaultKVStore = boltdb.Name +} diff --git a/vendor/github.com/blevesearch/bleve/config_forestdb.go b/vendor/github.com/blevesearch/bleve/config_forestdb.go deleted file mode 100644 index 206ad246..00000000 --- a/vendor/github.com/blevesearch/bleve/config_forestdb.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package bleve - -import ( - _ "github.com/blevesearch/bleve/index/store/forestdb" -) diff --git a/vendor/github.com/blevesearch/bleve/config_icu.go b/vendor/github.com/blevesearch/bleve/config_icu.go deleted file mode 100644 index c43e2866..00000000 --- a/vendor/github.com/blevesearch/bleve/config_icu.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build icu full - -package bleve - -import ( - _ "github.com/blevesearch/bleve/analysis/tokenizers/icu" -) diff --git a/vendor/github.com/blevesearch/bleve/config_kagome.go b/vendor/github.com/blevesearch/bleve/config_kagome.go deleted file mode 100644 index 58634c3f..00000000 --- a/vendor/github.com/blevesearch/bleve/config_kagome.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build kagome full - -package bleve - -import ( - _ "github.com/blevesearch/bleve/analysis/language/ja" -) diff --git a/vendor/github.com/blevesearch/bleve/config_leveldb.go b/vendor/github.com/blevesearch/bleve/config_leveldb.go deleted file mode 100644 index 0af9f59b..00000000 --- a/vendor/github.com/blevesearch/bleve/config_leveldb.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package bleve - -import ( - _ "github.com/blevesearch/bleve/index/store/leveldb" -) - -func init() { - // install leveldb as the default kv store - Config.DefaultKVStore = "leveldb" -} diff --git a/vendor/github.com/blevesearch/bleve/config_metrics.go b/vendor/github.com/blevesearch/bleve/config_metrics.go deleted file mode 100644 index 7b7ae581..00000000 --- a/vendor/github.com/blevesearch/bleve/config_metrics.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build debug - -package bleve - -import ( - _ "github.com/blevesearch/bleve/index/store/metrics" -) - diff --git a/vendor/github.com/blevesearch/bleve/config_rocksdb.go b/vendor/github.com/blevesearch/bleve/config_rocksdb.go deleted file mode 100644 index 0425c228..00000000 --- a/vendor/github.com/blevesearch/bleve/config_rocksdb.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package bleve - -import ( - _ "github.com/blevesearch/bleve/index/store/gorocksdb" -) diff --git a/vendor/github.com/blevesearch/bleve/config_stemmer.go b/vendor/github.com/blevesearch/bleve/config_stemmer.go deleted file mode 100644 index c99b8106..00000000 --- a/vendor/github.com/blevesearch/bleve/config_stemmer.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build libstemmer full - -package bleve - -import ( - _ "github.com/blevesearch/bleve/analysis/token_filters/stemmer_filter" - - _ "github.com/blevesearch/bleve/analysis/language/porter" -) diff --git a/vendor/github.com/blevesearch/bleve/doc.go b/vendor/github.com/blevesearch/bleve/doc.go index 017410ad..d54af7c9 100644 --- a/vendor/github.com/blevesearch/bleve/doc.go +++ b/vendor/github.com/blevesearch/bleve/doc.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. /* Package bleve is a library for indexing and searching text. diff --git a/vendor/github.com/blevesearch/bleve/docs/bleve.png b/vendor/github.com/blevesearch/bleve/docs/bleve.png deleted file mode 100644 index c2227aea27728a63536b64cfb2f57e0e7bd81040..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6727 zcmZ{n2UHVV_qUT6rHV8`niL^`KoW`|y%R#Onjk&2(53e#AiWm>0qLM3C@2U50i=o2 zi;DE#dv70j@Be-M?z-=+H8W@S-p|?lx96N$Nw|g@;yNKMApiilj#QM>y0}YSp7_@; zt}n5V#{d9AX{@ZQ22xg*NyEv(0&8my04QG1OmWxI{z3J$yH2RChn$6pZHLy2fT>SZ z@g`U^92uYi3Jr$1$z4^TB;$Nwrh@|_p0Z%%DT$s0nGO?dk!rLyYRbvajz-xKte4)~ z9!o!MLy!NMuNwYgx$Zk$1JF1L0ynv!fB^zID9W|vMtyRM&d>XxTckh|6c9IQVF5-Q z$bkk<>crF1iWOTc_ST>QX0;>S4H^*Vp>zLu79m;IYXCy|K|wJ;2=y!|NxPzjhVLBw zWha@FFO-*$Du*EVHOutpGTXhPIj-7{XQ_~x99nc2%_o7Zs9KG&bhgR7C~%JcGqwrK zJRdysIvr#>V0NFQzyIL53II--X7qI)l4BdrCr?SU7QO`n;3s_+{^Udwc_?g^1@I`e zoQVL3BTWm*hM!?)7uqe3HY>CbO?kPJnpG#mJXE@ODv4v@)B$AMD9@&g!t?$M|hO?Njk~uVdX!$VzvXbW|34+;-(Ptd0t>$EugXP*I<(3#hcaCN(?x zT78`fuyqwsIre<}cBrjk!sjVscB06Y6QHk4MbMgZRsVU21eUb}-hK=#fRR(nY=kni#%QeMLL`cGD+ z@nz1jd|qesM~p=J5%!ZR(k$9@d&IG3PHef{G6Mf#P(=_{3j9u5;`_xClRE5b&d1lpY_^)HxyNaq~GIHEv{5 zshUcHq^gN}l5PXQ9b(j!h%eQSf2CIeg-^FBCliNa1(Gq_#IxjsO61eyNb>1EMk~fY zp1E@EX^zRWTU4zy(@)V+lco<$hD?>L}9;i^y30jcy2Ob6$m=3-zM6th?bR!TX z%ZTRr{J=QY_)&T4+biEqRz&GZ+Hf>e^p1YjxAgh!@4W$6-58Fl4OnizJ_*_M_;h!S zay{rqqw|KT4|6@CKfV9$@mR*z2KG1tJ+MvC4dxUkR$EpbQd<0Z{L^5vCIQXBVs#gm zP*Ts&`lg(3W4=p&LefLj5-E|PtkhbpoN1MTI!J341Bf`JjybVI#0=qd3oT`+{)RQ1 z$%ySnf}}&5m1sF%-bGWFE!sz#q!!Ej*b10wW%>&|`>HEdD@A^q|AaFjgz}I36Bk={ z7#Hex2+Y~cAI);i($CV)R(#RA)&w^`;Mbxlh)RsAjf&kCnETj0-8~{$hBZ9@#F>0} z*Z!_gGE1^WGB7#zrHZb@Amw1@OS_lq18{4IwTbl)>*GP_tLW~cZuhM2Oe2BwnUkkW zOM8hNhFSs72pT?2W8I8p`5jhPeStfatm^&}yU-jP2)5{ZWy^s0uIAT*` z`Vpi&JQJCKjP6j?{ys3KV_ef+*-dOSX9W>>b+5K9xBM3x{aRt@g>bW2yHB2H#vvDx zcqA$^lWLU8hMNRVP(Dgkt5(~+N3zFq3_0e-6MyRQw2ox|yNeq~ZLsek+4906k$CK6cPVrDPXq1tMW6pr?NbPM7Tt>1Vz1D{VQMC zfypW8)a__=dw661X!uwid>{OpV1Y0ntN?z1CqcwVKz`K~+}sFktPNs>^%=414^V+* zPGsFe#0c&Y8{9l{z6IqLcnw+ctk(7L^Kb|An99q^ey&xFZSQMGL`G97Q@Ja2AX?-H z5vuau@~QI5sVHuA#f(CfUKCTII29w^O>b5=r;`1?QRS!8D^rBiYfygul!}#~PVZ++ zI=kP*88xCG-@Q%8$S_WN9!}9zGOO}t$LOQ`EOgZfBtM7-sq2fRC70IteM=cUyik7vQzxA(W%fG zZB93CA2Bwd`9ib2-q@;QeXdpJqm1Tc9Z%OI65AHro3^W?78Rw|BdIKgy}PEJQxQ|@ zY`+qg$I-qI4@bZ51+Uf>>=hilPq^W`{~BGa*+!+Aq78rayqPd4o&CC%yH!>DaHal% z$)m@{H?Xm0v6Xzq@9LDwtPz7nOZiKIw6W=PbxYn=tC>IBMur>4?fmG~3 zeLMZQ&Ea%nSJs>2)uIM}-p2#*eP-d!skQ^g^(PFwuqiu!?ia(1+&yAQ$1mIuVnVs3mtc>9=71nqb6 zo07m$5$y~BK<`|hK#u}RHvj<4iq%26pj1^vFb;N56Eg=>bEt=%<3%(8Am$-*F|;#x zF=6tsv$b~?@epVE9U*ctzBI#Fn0|-2*od>BR5h4n9h}UW?m-2ha25$dCMG5^Co>BX zEjfk1@rx&M7AqGQM-dpz-Q6AP&JT5PvV`#p3k$>Gd@w#fo{I<`XHR<<6AvDH=R1E* z@_+NlnLA^gu#PTR2YaT=c}+|mTwTOjSS}O&=lH8m7p%p9Gub=;E$c!c?9v0{g~DO~ zhM9X{{|9#I`Jb2>=0B<&U7c)yYcj*Y%x%r>%w4J&U3;vH1h3d{LH{)Wwx+x|0(KE+dt#c|C-@X>mS5r^F=hU9_F?vIqXGCf9ntw z5QF`z|!m5s5+QoEj;B+T+Ai-;JgANa3K+XUM9Hk<>3B>+<(OUQNg80 z*2&z&#lcC(!NFGIw+g$ zzLGLUo)}0+25u^)X{!60@&Q|60--0VHjqqthMC?$%PLF^93$GKy%zIC)NOswf-&(t zdo(L^G;K8V<4E26S9=zPMMcNw^=HTDHQsad9HvJ%ee#NM26b1ol*ztvsP)4dg{&oC zs;xh12<(o;$j9rQ-UC7=*wu1n<`}b`4SByl-f_(%c3}})K8;&??yeSe8cH`IK0LGE z%TjmrVY3 zrW-v%)U${vW&gUCI2q>-@***e>#bs)Nd_=@q=;yX!*G##p~0Y6vSSz3bZZqUcmumu zc%@|m+4x`=l~_jYSu{pE(USKKFdOyH8B!PjU*B+-)w?R`KN-Vvw;2#FIR3jXq$>T$R z$(BpcF|yQmz&V@q=GV4E)!|9<^yq)Dms&lKygSCjID0&{xv$sH2;UIUOnXS|JOL zp=VFv&`4JbpN#vEB*+zyei|^gie7rH;(!4&ul4a=t;c=vy4iPbB-{mBV(y%c?>aGQtjfSgZcIG(HQ~WCF72`hnx9PBBjaU7nKG@I zB&LGDD_*m+7YbrrQ8-xfbUKFXCqL2Tdkpg7U(;+4OKf9JM+Sjy$gzq6B%f$Xlq=%B zf1$ObrOUXVj1I>FT1bh&*u7L2J9GkR-4IDee|UScATer-n|}6ml15xc6YP!UR9dT! z&YGRDf?ySyGQNQo7=WP@s`AG=S3b7MX48jQ>M&TJ??F}!VNz0dF=mN-er&}Cwb#z( zv+@nIBY-Y5GaTETFOEk&>0V|%lNCnxC4tcARs)S-0;El`7K2B>mZdFO>>|#4M00O& z@m8g&u>N~lD@E4y{2bDwgIh}LX;BHVQ9{f2Ms?I!yF5bkgL(0*!>U4ko?WO+i-6Ez zdh!x-oK{i&+m4#Rw4szWElB^gxWyd=Np|gvq++k5#{=9(&$(4Anr#}4B8PO#*tf|o zGk1?Qb{F*7sN~AFK6lvC_dH+=ed3peL&+mU0BeV}#BL6GDMI`l8@dmg#9cVzae8V% zSnsr0R~-hywfbcTDixzJ>1`|mh3P99zEk}!%ilsOhC!HYEORA)TEnmxQ*lwSzP~HN z3kiJ7X92|;T_5Ke*rq!T8xn@7hOTpoH9s1h&U}ulCVFDzO?FQ?RqDgYlzrLz@UDVT zNaZ<}0Xa0f5k(*VLFlYFrQG{hv5$}7eTi~YFp_1J1ZGPw`%I$~N&HloP_VTfMZ=G& zO04+Vj=f%Fuyg<)Exz81)9u^MEcWyxvmQR7D$p(}R_|o?qL2$8d~rz4Mf%Z&W^B_) z>cD?~kgo}I|BlN6K1kFpcdE8xf>tapTP3BlKg|~iLXb##Z4lD4=Xtwvt)c2XfVKq# zS5k5vez6#ixHBA;^?4^to=AMTot~3UE8;^L_-zEcIUVA%MyzM(aWh^$W0GD9*u(U` zY(xs^Bz8iSrBJy_n_ND|78Upw4!La|75{b(u`k=r?5I<+i3Z1Lw|?bv%#K=liI_OG zD>f7uj|00k*~YYkgXy}N`n=@vkTI=}yYBv>(GUFH)M?(7`VuvcdRxu8iN%Jn zHt(3pa1#o(M@6FGVODq)#tX$AcrKO%8^IyGyn?ML^@~nXKjeHb6Z9rCiCDCWCp&j} z0-*#|_qo6DIDx=;zN=1PIdNj+AzAUOB&_T(f^oW!Vms;3Kd!Lz?H);i&*DIbW5TjK z!vQI8+LM>bX2e;dM@0l1`WZ?9qgx5(_XjDeZ#eBf;YReFrTRN1$2FVidpQC1485+C zNAD`y5XEmZi@7#0LZh)>G(X>()X9X&bz)NFGuraRxX!>IR$d5dtK6*OqYoJ=+7o+1 zEe^Xq@C|-IbbZWe4kmbzclFF8(8yq$%hoS4qcyFVeYW`PFaSX}*`2){`_QtN93!l`#plCGW1 z<&V$>&i9Fc?)S$93s>|amDQYCd#F(N!`eSu>}21g7=w)kb(P{VXF9OrL**W&lSw!m zuIfU#Rr)wG9bOo}*Wm1SbF?s8Tq}Mah#1^%^m>B3qV>A7=yQ)bl=ynv z7k62o(akc|YWO}P%++o2%hu@tD{lLl(D3%u;cKuXGbC^vw?Ps~PC=;-R|f6safdFi z*Tq^nbA4evqgn;2Gi$jef>_-HJzp*}@Y2c-y)LW^bhbk8@j+o1pNNX889q%S3E?^P zQW%N+5*U?-fi`Gm^XhY^|74h&XsHE`I^Lj-*g}tMvg#+29Asoke zC%roP713MbJDBay>i6WGO^GE&o(+q=ex42qP+nG?h)&@4zWNF9?%|I52P6e^)|Cyl zxNozp(xlpt`0l)wn#yG9p_R!;A()yWSddtU3r?<*dbfZvk*8q1!_h7UOkj3a-P<`9dPy}i&PNru(S>&c1zC0(W!fW#yxXE@$&AH$v6Zwd z5^m(0>0)g2j7_WGlnp+#ddmSGTBQUJoQmGG>A8(o#w=e!r?RU zxv+EFN=A;Z=olZP!^fVrdV(eApiizP{NT9u6<*+p&pIn#26}P7a-vk(z{~FEvLPAn z<|ZPF(*{p@8(hkn))n1YF_6}KsPaDI#E=4alZFH@l8D1ZV~H1-RiIXpRh7G4vb$Ke zB?EZ)>P`;ct9G4jfeG)N?Go3evMmL`Lp#NwkgGa<<${7 z!`qRBH?mg0i2~D*@QMI(@ob|KOa@;;5GgMDEj#LwymjOJ!zI4J`I8_rTyjRLB4*<- zLF5CQ_FWF(uVNDw#dyW^wY$fAA&E6A6T%G>V8=Tm~NB)mlK^cKhLYj%xW+<|#FS@p)zOw^4~d9UQO acc.out -for Dir in . $(find ./* -maxdepth 10 -type d ); -do - if ls $Dir/*.go &> /dev/null; - then - returnval=`go test -coverprofile=profile.out -covermode=count $Dir` - echo ${returnval} - if [[ ${returnval} != *FAIL* ]] - then - if [ -f profile.out ] - then - cat profile.out | grep -v "mode: count" >> acc.out - fi - else - exit 1 - fi - fi -done - -# collect integration test coverage -echo "mode: count" > integration-acc.out -INTPACKS=`go list ./... | grep -v utils | xargs | sed 's/ /,/g'` -returnval=`go test -coverpkg=$INTPACKS -coverprofile=profile.out -covermode=count ./test` -if [[ ${returnval} != *FAIL* ]] -then - if [ -f profile.out ] - then - cat profile.out | grep -v "mode: count" >> integration-acc.out - fi -else - exit 1 -fi - -cat acc.out integration-acc.out | go run docs/merge-coverprofile.go > merged.out - -if [ -n "$COVERALLS" ] -then - goveralls -service drone.io -coverprofile=merged.out -repotoken $COVERALLS -fi - -if [ -n "$COVERHTML" ] -then - go tool cover -html=merged.out -fi - -rm -rf ./profile.out -rm -rf ./acc.out -rm -rf ./integration-acc.out -rm -rf ./merged.out \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/document/document.go b/vendor/github.com/blevesearch/bleve/document/document.go index 525ceb31..ed36b127 100644 --- a/vendor/github.com/blevesearch/bleve/document/document.go +++ b/vendor/github.com/blevesearch/bleve/document/document.go @@ -1,22 +1,26 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package document -import ( - "fmt" -) +import "fmt" type Document struct { ID string `json:"id"` Fields []Field `json:"fields"` CompositeFields []*CompositeField + Number uint64 `json:"-"` } func NewDocument(id string) *Document { @@ -54,3 +58,18 @@ func (d *Document) GoString() string { } return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID, fields, compositeFields) } + +func (d *Document) NumPlainTextBytes() uint64 { + rv := uint64(0) + for _, field := range d.Fields { + rv += field.NumPlainTextBytes() + } + for _, compositeField := range d.CompositeFields { + for _, field := range d.Fields { + if compositeField.includesField(field.Name()) { + rv += field.NumPlainTextBytes() + } + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/document/field.go b/vendor/github.com/blevesearch/bleve/document/field.go index 3d46d080..c17f81e5 100644 --- a/vendor/github.com/blevesearch/bleve/document/field.go +++ b/vendor/github.com/blevesearch/bleve/document/field.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package document @@ -14,9 +19,21 @@ import ( ) type Field interface { + // Name returns the path of the field from the root DocumentMapping. + // A root field path is "field", a subdocument field is "parent.field". Name() string + // ArrayPositions returns the intermediate document and field indices + // required to resolve the field value in the document. For example, if the + // field path is "doc1.doc2.field" where doc1 and doc2 are slices or + // arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in + // "doc1", then "field" in "doc2". ArrayPositions() []uint64 Options() IndexingOptions Analyze() (int, analysis.TokenFrequencies) Value() []byte + + // NumPlainTextBytes should return the number of plain text bytes + // that this field represents - this is a common metric for tracking + // the rate of indexing + NumPlainTextBytes() uint64 } diff --git a/vendor/github.com/blevesearch/bleve/document/field_boolean.go b/vendor/github.com/blevesearch/bleve/document/field_boolean.go new file mode 100644 index 00000000..668b431a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/document/field_boolean.go @@ -0,0 +1,107 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + + "github.com/blevesearch/bleve/analysis" +) + +const DefaultBooleanIndexingOptions = StoreField | IndexField + +type BooleanField struct { + name string + arrayPositions []uint64 + options IndexingOptions + value []byte + numPlainTextBytes uint64 +} + +func (b *BooleanField) Name() string { + return b.name +} + +func (b *BooleanField) ArrayPositions() []uint64 { + return b.arrayPositions +} + +func (b *BooleanField) Options() IndexingOptions { + return b.options +} + +func (b *BooleanField) Analyze() (int, analysis.TokenFrequencies) { + tokens := make(analysis.TokenStream, 0) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(b.value), + Term: b.value, + Position: 1, + Type: analysis.Boolean, + }) + + fieldLength := len(tokens) + tokenFreqs := analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors()) + return fieldLength, tokenFreqs +} + +func (b *BooleanField) Value() []byte { + return b.value +} + +func (b *BooleanField) Boolean() (bool, error) { + if len(b.value) == 1 { + return b.value[0] == 'T', nil + } + return false, fmt.Errorf("boolean field has %d bytes", len(b.value)) +} + +func (b *BooleanField) GoString() string { + return fmt.Sprintf("&document.BooleanField{Name:%s, Options: %s, Value: %s}", b.name, b.options, b.value) +} + +func (b *BooleanField) NumPlainTextBytes() uint64 { + return b.numPlainTextBytes +} + +func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField { + return &BooleanField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField { + return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions) +} + +func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField { + numPlainTextBytes := 5 + v := []byte("F") + if b { + numPlainTextBytes = 4 + v = []byte("T") + } + return &BooleanField{ + name: name, + arrayPositions: arrayPositions, + value: v, + options: options, + numPlainTextBytes: uint64(numPlainTextBytes), + } +} diff --git a/vendor/github.com/blevesearch/bleve/document/field_composite.go b/vendor/github.com/blevesearch/bleve/document/field_composite.go index 94c68d9d..b41b1b8e 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_composite.go +++ b/vendor/github.com/blevesearch/bleve/document/field_composite.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package document @@ -31,11 +36,12 @@ func NewCompositeField(name string, defaultInclude bool, include []string, exclu func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options IndexingOptions) *CompositeField { rv := &CompositeField{ - name: name, - options: options, - defaultInclude: defaultInclude, - includedFields: make(map[string]bool, len(include)), - excludedFields: make(map[string]bool, len(exclude)), + name: name, + options: options, + defaultInclude: defaultInclude, + includedFields: make(map[string]bool, len(include)), + excludedFields: make(map[string]bool, len(exclude)), + compositeFrequencies: make(analysis.TokenFrequencies), } for _, i := range include { @@ -68,7 +74,11 @@ func (c *CompositeField) Value() []byte { return []byte{} } -func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) { +func (c *CompositeField) NumPlainTextBytes() uint64 { + return 0 +} + +func (c *CompositeField) includesField(field string) bool { shouldInclude := c.defaultInclude _, fieldShouldBeIncluded := c.includedFields[field] if fieldShouldBeIncluded { @@ -78,9 +88,12 @@ func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFr if fieldShouldBeExcluded { shouldInclude = false } + return shouldInclude +} - if shouldInclude { +func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) { + if c.includesField(field) { c.totalLength += length - c.compositeFrequencies = c.compositeFrequencies.MergeAll(field, freq) + c.compositeFrequencies.MergeAll(field, freq) } } diff --git a/vendor/github.com/blevesearch/bleve/document/field_datetime.go b/vendor/github.com/blevesearch/bleve/document/field_datetime.go index 9ecbfbe0..6783d53d 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_datetime.go +++ b/vendor/github.com/blevesearch/bleve/document/field_datetime.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package document @@ -15,7 +20,7 @@ import ( "time" "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/numeric_util" + "github.com/blevesearch/bleve/numeric" ) const DefaultDateTimeIndexingOptions = StoreField | IndexField @@ -25,10 +30,11 @@ var MinTimeRepresentable = time.Unix(0, math.MinInt64) var MaxTimeRepresentable = time.Unix(0, math.MaxInt64) type DateTimeField struct { - name string - arrayPositions []uint64 - options IndexingOptions - value numeric_util.PrefixCoded + name string + arrayPositions []uint64 + options IndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 } func (n *DateTimeField) Name() string { @@ -58,7 +64,7 @@ func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) { shift := DefaultDateTimePrecisionStep for shift < 64 { - shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift) + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) if err != nil { break } @@ -75,7 +81,7 @@ func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) { } fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens) + tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) return fieldLength, tokenFreqs } @@ -95,12 +101,17 @@ func (n *DateTimeField) GoString() string { return fmt.Sprintf("&document.DateField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) } +func (n *DateTimeField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byte) *DateTimeField { return &DateTimeField{ - name: name, - arrayPositions: arrayPositions, - value: value, - options: DefaultDateTimeIndexingOptions, + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultDateTimeIndexingOptions, + numPlainTextBytes: uint64(len(value)), } } @@ -111,12 +122,15 @@ func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*Date func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options IndexingOptions) (*DateTimeField, error) { if canRepresent(dt) { dtInt64 := dt.UnixNano() - prefixCoded := numeric_util.MustNewPrefixCodedInt64(dtInt64, 0) + prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0) return &DateTimeField{ name: name, arrayPositions: arrayPositions, value: prefixCoded, options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), }, nil } return nil, fmt.Errorf("cannot represent %s in this type", dt) diff --git a/vendor/github.com/blevesearch/bleve/document/field_geopoint.go b/vendor/github.com/blevesearch/bleve/document/field_geopoint.go new file mode 100644 index 00000000..f508b362 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/document/field_geopoint.go @@ -0,0 +1,137 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package document + +import ( + "fmt" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/geo" + "github.com/blevesearch/bleve/numeric" +) + +var GeoPrecisionStep uint = 9 + +type GeoPointField struct { + name string + arrayPositions []uint64 + options IndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 +} + +func (n *GeoPointField) Name() string { + return n.name +} + +func (n *GeoPointField) ArrayPositions() []uint64 { + return n.arrayPositions +} + +func (n *GeoPointField) Options() IndexingOptions { + return n.options +} + +func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) { + tokens := make(analysis.TokenStream, 0) + tokens = append(tokens, &analysis.Token{ + Start: 0, + End: len(n.value), + Term: n.value, + Position: 1, + Type: analysis.Numeric, + }) + + original, err := n.value.Int64() + if err == nil { + + shift := GeoPrecisionStep + for shift < 64 { + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) + if err != nil { + break + } + token := analysis.Token{ + Start: 0, + End: len(shiftEncoded), + Term: shiftEncoded, + Position: 1, + Type: analysis.Numeric, + } + tokens = append(tokens, &token) + shift += GeoPrecisionStep + } + } + + fieldLength := len(tokens) + tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) + return fieldLength, tokenFreqs +} + +func (n *GeoPointField) Value() []byte { + return n.value +} + +func (n *GeoPointField) Lon() (float64, error) { + i64, err := n.value.Int64() + if err != nil { + return 0.0, err + } + return geo.MortonUnhashLon(uint64(i64)), nil +} + +func (n *GeoPointField) Lat() (float64, error) { + i64, err := n.value.Int64() + if err != nil { + return 0.0, err + } + return geo.MortonUnhashLat(uint64(i64)), nil +} + +func (n *GeoPointField) GoString() string { + return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) +} + +func (n *GeoPointField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + +func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField { + return &GeoPointField{ + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), + } +} + +func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField { + return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions) +} + +func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField { + mhash := geo.MortonHash(lon, lat) + prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0) + return &GeoPointField{ + name: name, + arrayPositions: arrayPositions, + value: prefixCoded, + options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), + } +} diff --git a/vendor/github.com/blevesearch/bleve/document/field_numeric.go b/vendor/github.com/blevesearch/bleve/document/field_numeric.go index 38ef8578..7faae2bb 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_numeric.go +++ b/vendor/github.com/blevesearch/bleve/document/field_numeric.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package document @@ -13,7 +18,7 @@ import ( "fmt" "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/numeric_util" + "github.com/blevesearch/bleve/numeric" ) const DefaultNumericIndexingOptions = StoreField | IndexField @@ -21,10 +26,11 @@ const DefaultNumericIndexingOptions = StoreField | IndexField const DefaultPrecisionStep uint = 4 type NumericField struct { - name string - arrayPositions []uint64 - options IndexingOptions - value numeric_util.PrefixCoded + name string + arrayPositions []uint64 + options IndexingOptions + value numeric.PrefixCoded + numPlainTextBytes uint64 } func (n *NumericField) Name() string { @@ -54,7 +60,7 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) { shift := DefaultPrecisionStep for shift < 64 { - shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift) + shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift) if err != nil { break } @@ -71,7 +77,7 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) { } fieldLength := len(tokens) - tokenFreqs := analysis.TokenFrequency(tokens) + tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors()) return fieldLength, tokenFreqs } @@ -84,19 +90,24 @@ func (n *NumericField) Number() (float64, error) { if err != nil { return 0.0, err } - return numeric_util.Int64ToFloat64(i64), nil + return numeric.Int64ToFloat64(i64), nil } func (n *NumericField) GoString() string { return fmt.Sprintf("&document.NumericField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value) } +func (n *NumericField) NumPlainTextBytes() uint64 { + return n.numPlainTextBytes +} + func NewNumericFieldFromBytes(name string, arrayPositions []uint64, value []byte) *NumericField { return &NumericField{ - name: name, - arrayPositions: arrayPositions, - value: value, - options: DefaultNumericIndexingOptions, + name: name, + arrayPositions: arrayPositions, + value: value, + options: DefaultNumericIndexingOptions, + numPlainTextBytes: uint64(len(value)), } } @@ -105,12 +116,15 @@ func NewNumericField(name string, arrayPositions []uint64, number float64) *Nume } func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options IndexingOptions) *NumericField { - numberInt64 := numeric_util.Float64ToInt64(number) - prefixCoded := numeric_util.MustNewPrefixCodedInt64(numberInt64, 0) + numberInt64 := numeric.Float64ToInt64(number) + prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0) return &NumericField{ name: name, arrayPositions: arrayPositions, value: prefixCoded, options: options, + // not correct, just a place holder until we revisit how fields are + // represented and can fix this better + numPlainTextBytes: uint64(8), } } diff --git a/vendor/github.com/blevesearch/bleve/document/field_numeric_test.go b/vendor/github.com/blevesearch/bleve/document/field_numeric_test.go deleted file mode 100644 index df40c588..00000000 --- a/vendor/github.com/blevesearch/bleve/document/field_numeric_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package document - -import ( - "testing" -) - -func TestNumericField(t *testing.T) { - nf := NewNumericField("age", []uint64{}, 3.4) - numTokens, tokenFreqs := nf.Analyze() - if numTokens != 16 { - t.Errorf("expected 16 tokens") - } - if len(tokenFreqs) != 16 { - t.Errorf("expected 16 token freqs") - } -} diff --git a/vendor/github.com/blevesearch/bleve/document/field_text.go b/vendor/github.com/blevesearch/bleve/document/field_text.go index ff59aa4a..37873d36 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_text.go +++ b/vendor/github.com/blevesearch/bleve/document/field_text.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package document @@ -18,11 +23,12 @@ import ( const DefaultTextIndexingOptions = IndexField type TextField struct { - name string - arrayPositions []uint64 - options IndexingOptions - analyzer *analysis.Analyzer - value []byte + name string + arrayPositions []uint64 + options IndexingOptions + analyzer *analysis.Analyzer + value []byte + numPlainTextBytes uint64 } func (t *TextField) Name() string { @@ -60,7 +66,7 @@ func (t *TextField) Analyze() (int, analysis.TokenFrequencies) { } } fieldLength := len(tokens) // number of tokens in this doc field - tokenFreqs := analysis.TokenFrequency(tokens) + tokenFreqs := analysis.TokenFrequency(tokens, t.arrayPositions, t.options.IncludeTermVectors()) return fieldLength, tokenFreqs } @@ -69,7 +75,11 @@ func (t *TextField) Value() []byte { } func (t *TextField) GoString() string { - return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %s, Value: %s}", t.name, t.options, t.analyzer, t.value) + return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}", t.name, t.options, t.analyzer, t.value, t.arrayPositions) +} + +func (t *TextField) NumPlainTextBytes() uint64 { + return t.numPlainTextBytes } func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField { @@ -78,29 +88,32 @@ func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options IndexingOptions) *TextField { return &TextField{ - name: name, - arrayPositions: arrayPositions, - options: options, - value: value, + name: name, + arrayPositions: arrayPositions, + options: options, + value: value, + numPlainTextBytes: uint64(len(value)), } } func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte, analyzer *analysis.Analyzer) *TextField { return &TextField{ - name: name, - arrayPositions: arrayPositions, - options: DefaultTextIndexingOptions, - analyzer: analyzer, - value: value, + name: name, + arrayPositions: arrayPositions, + options: DefaultTextIndexingOptions, + analyzer: analyzer, + value: value, + numPlainTextBytes: uint64(len(value)), } } func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options IndexingOptions, analyzer *analysis.Analyzer) *TextField { return &TextField{ - name: name, - arrayPositions: arrayPositions, - options: options, - analyzer: analyzer, - value: value, + name: name, + arrayPositions: arrayPositions, + options: options, + analyzer: analyzer, + value: value, + numPlainTextBytes: uint64(len(value)), } } diff --git a/vendor/github.com/blevesearch/bleve/document/indexing_options.go b/vendor/github.com/blevesearch/bleve/document/indexing_options.go index 63c4c7c9..5d562c1d 100644 --- a/vendor/github.com/blevesearch/bleve/document/indexing_options.go +++ b/vendor/github.com/blevesearch/bleve/document/indexing_options.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package document diff --git a/vendor/github.com/blevesearch/bleve/document/indexing_options_test.go b/vendor/github.com/blevesearch/bleve/document/indexing_options_test.go deleted file mode 100644 index 3357de32..00000000 --- a/vendor/github.com/blevesearch/bleve/document/indexing_options_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package document - -import ( - "testing" -) - -func TestIndexingOptions(t *testing.T) { - tests := []struct { - options IndexingOptions - isIndexed bool - isStored bool - includeTermVectors bool - }{ - { - options: IndexField | StoreField | IncludeTermVectors, - isIndexed: true, - isStored: true, - includeTermVectors: true, - }, - { - options: IndexField | IncludeTermVectors, - isIndexed: true, - isStored: false, - includeTermVectors: true, - }, - { - options: StoreField | IncludeTermVectors, - isIndexed: false, - isStored: true, - includeTermVectors: true, - }, - { - options: IndexField, - isIndexed: true, - isStored: false, - includeTermVectors: false, - }, - { - options: StoreField, - isIndexed: false, - isStored: true, - includeTermVectors: false, - }, - } - - for _, test := range tests { - actuallyIndexed := test.options.IsIndexed() - if actuallyIndexed != test.isIndexed { - t.Errorf("expected indexed to be %v, got %v for %d", test.isIndexed, actuallyIndexed, test.options) - } - actuallyStored := test.options.IsStored() - if actuallyStored != test.isStored { - t.Errorf("expected stored to be %v, got %v for %d", test.isStored, actuallyStored, test.options) - } - actuallyIncludeTermVectors := test.options.IncludeTermVectors() - if actuallyIncludeTermVectors != test.includeTermVectors { - t.Errorf("expected includeTermVectors to be %v, got %v for %d", test.includeTermVectors, actuallyIncludeTermVectors, test.options) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/error.go b/vendor/github.com/blevesearch/bleve/error.go index 52d845e3..7402dfcf 100644 --- a/vendor/github.com/blevesearch/bleve/error.go +++ b/vendor/github.com/blevesearch/bleve/error.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve @@ -15,15 +20,13 @@ const ( ErrorIndexPathDoesNotExist ErrorIndexMetaMissing ErrorIndexMetaCorrupt - ErrorDisjunctionFewerThanMinClauses - ErrorBooleanQueryNeedsMustOrShould - ErrorNumericQueryNoBounds - ErrorPhraseQueryNoTerms - ErrorUnknownQueryType ErrorUnknownStorageType ErrorIndexClosed ErrorAliasMulti ErrorAliasEmpty + ErrorUnknownIndexType + ErrorEmptyID + ErrorIndexReadInconsistency ) // Error represents a more strongly typed bleve error for detecting @@ -31,21 +34,19 @@ const ( type Error int func (e Error) Error() string { - return errorMessages[int(e)] + return errorMessages[e] } -var errorMessages = map[int]string{ - int(ErrorIndexPathExists): "cannot create new index, path already exists", - int(ErrorIndexPathDoesNotExist): "cannot open index, path does not exist", - int(ErrorIndexMetaMissing): "cannot open index, metadata missing", - int(ErrorIndexMetaCorrupt): "cannot open index, metadata corrupt", - int(ErrorDisjunctionFewerThanMinClauses): "disjunction query has fewer than the minimum number of clauses to satisfy", - int(ErrorBooleanQueryNeedsMustOrShould): "boolean query must contain at least one must or should clause", - int(ErrorNumericQueryNoBounds): "numeric range query must specify min or max", - int(ErrorPhraseQueryNoTerms): "phrase query must contain at least one term", - int(ErrorUnknownQueryType): "unknown query type", - int(ErrorUnknownStorageType): "unknown storage type", - int(ErrorIndexClosed): "index is closed", - int(ErrorAliasMulti): "cannot perform single index operation on multiple index alias", - int(ErrorAliasEmpty): "cannot perform operation on empty alias", +var errorMessages = map[Error]string{ + ErrorIndexPathExists: "cannot create new index, path already exists", + ErrorIndexPathDoesNotExist: "cannot open index, path does not exist", + ErrorIndexMetaMissing: "cannot open index, metadata missing", + ErrorIndexMetaCorrupt: "cannot open index, metadata corrupt", + ErrorUnknownStorageType: "unknown storage type", + ErrorIndexClosed: "index is closed", + ErrorAliasMulti: "cannot perform single index operation on multiple index alias", + ErrorAliasEmpty: "cannot perform operation on empty alias", + ErrorUnknownIndexType: "unknown index type", + ErrorEmptyID: "document ID cannot be empty", + ErrorIndexReadInconsistency: "index read inconsistency detected", } diff --git a/vendor/github.com/blevesearch/bleve/examples_test.go b/vendor/github.com/blevesearch/bleve/examples_test.go deleted file mode 100644 index 6abef9aa..00000000 --- a/vendor/github.com/blevesearch/bleve/examples_test.go +++ /dev/null @@ -1,485 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "fmt" - "os" - "testing" - "time" -) - -var mapping *IndexMapping -var example_index Index -var err error - -func TestMain(m *testing.M) { - err = os.RemoveAll("path_to_index") - if err != nil { - panic(err) - } - toRun := m.Run() - err = os.RemoveAll("path_to_index") - if err != nil { - panic(err) - } - os.Exit(toRun) -} - -func ExampleNew() { - mapping = NewIndexMapping() - example_index, err = New("path_to_index", mapping) - if err != nil { - panic(err) - } - count, err := example_index.DocCount() - if err != nil { - panic(err) - } - - fmt.Println(count) - // Output: - // 0 -} - -func ExampleIndex_indexing() { - data := struct { - Name string - Created time.Time - }{Name: "named one", Created: time.Now()} - data2 := struct { - Name string - Created time.Time - }{Name: "great nameless one", Created: time.Now()} - - // index some data - err = example_index.Index("document id 1", data) - if err != nil { - panic(err) - } - err = example_index.Index("document id 2", data2) - if err != nil { - panic(err) - } - - // 2 documents have been indexed - count, err := example_index.DocCount() - if err != nil { - panic(err) - } - - fmt.Println(count) - // Output: - // 2 -} - -// Examples for query related functions - -func ExampleNewMatchQuery() { - // finds documents with fields fully matching the given query text - query := NewMatchQuery("named one") - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 1 -} - -func ExampleNewMatchAllQuery() { - // finds all documents in the index - query := NewMatchAllQuery() - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(len(searchResults.Hits)) - // Output: - // 2 -} - -func ExampleNewMatchNoneQuery() { - // matches no documents in the index - query := NewMatchNoneQuery() - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(len(searchResults.Hits)) - // Output: - // 0 -} - -func ExampleNewMatchPhraseQuery() { - // finds all documents with the given phrase in the index - query := NewMatchPhraseQuery("nameless one") - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 2 -} - -func ExampleNewNumericRangeQuery() { - value1 := float64(11) - value2 := float64(100) - data := struct{ Priority float64 }{Priority: float64(15)} - data2 := struct{ Priority float64 }{Priority: float64(10)} - - err = example_index.Index("document id 3", data) - if err != nil { - panic(err) - } - err = example_index.Index("document id 4", data2) - if err != nil { - panic(err) - } - - query := NewNumericRangeQuery(&value1, &value2) - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 3 -} - -func ExampleNewNumericRangeInclusiveQuery() { - value1 := float64(10) - value2 := float64(100) - v1incl := false - v2incl := false - - query := NewNumericRangeInclusiveQuery(&value1, &value2, &v1incl, &v2incl) - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 3 -} - -func ExampleNewPhraseQuery() { - // finds all documents with the given phrases in the given field in the index - query := NewPhraseQuery([]string{"nameless", "one"}, "Name") - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 2 -} - -func ExampleNewPrefixQuery() { - // finds all documents with terms having the given prefix in the index - query := NewPrefixQuery("name") - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(len(searchResults.Hits)) - // Output: - // 2 -} - -func ExampleNewQueryStringQuery() { - query := NewQueryStringQuery("+one -great") - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 1 -} - -func ExampleNewTermQuery() { - query := NewTermQuery("great") - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 2 -} - -func ExampleNewFacetRequest() { - facet := NewFacetRequest("Name", 1) - query := NewMatchAllQuery() - searchRequest := NewSearchRequest(query) - searchRequest.AddFacet("facet name", facet) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - // total number of terms - fmt.Println(searchResults.Facets["facet name"].Total) - // numer of docs with no value for this field - fmt.Println(searchResults.Facets["facet name"].Missing) - // term with highest occurences in field name - fmt.Println(searchResults.Facets["facet name"].Terms[0].Term) - // Output: - // 5 - // 2 - // one -} - -func ExampleFacetRequest_AddDateTimeRange() { - facet := NewFacetRequest("Created", 1) - facet.AddDateTimeRange("range name", time.Unix(0, 0), time.Now()) - query := NewMatchAllQuery() - searchRequest := NewSearchRequest(query) - searchRequest.AddFacet("facet name", facet) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - // dates in field Created since starting of unix time till now - fmt.Println(searchResults.Facets["facet name"].DateRanges[0].Count) - // Output: - // 2 -} - -func ExampleFacetRequest_AddNumericRange() { - value1 := float64(11) - - facet := NewFacetRequest("Priority", 1) - facet.AddNumericRange("range name", &value1, nil) - query := NewMatchAllQuery() - searchRequest := NewSearchRequest(query) - searchRequest.AddFacet("facet name", facet) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - // number documents with field Priority in the given range - fmt.Println(searchResults.Facets["facet name"].NumericRanges[0].Count) - // Output: - // 1 -} - -func ExampleNewHighlight() { - query := NewMatchQuery("nameless") - searchRequest := NewSearchRequest(query) - searchRequest.Highlight = NewHighlight() - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) - // Output: - // great nameless one -} - -func ExampleNewHighlightWithStyle() { - query := NewMatchQuery("nameless") - searchRequest := NewSearchRequest(query) - searchRequest.Highlight = NewHighlightWithStyle("ansi") - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) - // Output: - // great nameless one -} - -func ExampleSearchRequest_AddFacet() { - facet := NewFacetRequest("Name", 1) - query := NewMatchAllQuery() - searchRequest := NewSearchRequest(query) - searchRequest.AddFacet("facet name", facet) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - // total number of terms - fmt.Println(searchResults.Facets["facet name"].Total) - // numer of docs with no value for this field - fmt.Println(searchResults.Facets["facet name"].Missing) - // term with highest occurences in field name - fmt.Println(searchResults.Facets["facet name"].Terms[0].Term) - // Output: - // 5 - // 2 - // one -} - -func ExampleNewSearchRequest() { - // finds documents with fields fully matching the given query text - query := NewMatchQuery("named one") - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 1 -} - -func ExampleNewBooleanQuery() { - must := make([]Query, 1) - mustNot := make([]Query, 1) - must[0] = NewMatchQuery("one") - mustNot[0] = NewMatchQuery("great") - query := NewBooleanQuery(must, nil, mustNot) - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 1 -} - -func ExampleNewBooleanQueryMinShould() { - should := make([]Query, 2) - should[0] = NewMatchQuery("great") - should[1] = NewMatchQuery("one") - query := NewBooleanQueryMinShould(nil, should, nil, float64(2)) - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 2 -} - -func ExampleNewConjunctionQuery() { - conjuncts := make([]Query, 2) - conjuncts[0] = NewMatchQuery("great") - conjuncts[1] = NewMatchQuery("one") - query := NewConjunctionQuery(conjuncts) - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(searchResults.Hits[0].ID) - // Output: - // document id 2 -} - -func ExampleNewDisjunctionQuery() { - disjuncts := make([]Query, 2) - disjuncts[0] = NewMatchQuery("great") - disjuncts[1] = NewMatchQuery("named") - query := NewDisjunctionQuery(disjuncts) - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(len(searchResults.Hits)) - // Output: - // 2 -} - -func ExampleNewDisjunctionQueryMin() { - disjuncts := make([]Query, 2) - disjuncts[0] = NewMatchQuery("great") - disjuncts[1] = NewMatchQuery("named") - query := NewDisjunctionQueryMin(disjuncts, float64(2)) - searchRequest := NewSearchRequest(query) - searchResults, err := example_index.Search(searchRequest) - if err != nil { - panic(err) - } - - fmt.Println(len(searchResults.Hits)) - // Output: - // 0 -} - -// Examples for Mapping related functions - -func ExampleDocumentMapping_AddSubDocumentMapping() { - // adds a document mapping for a property in a document - // useful for mapping nested documents - documentMapping := NewDocumentMapping() - subDocumentMapping := NewDocumentMapping() - documentMapping.AddSubDocumentMapping("Property", subDocumentMapping) - - fmt.Println(len(documentMapping.Properties)) - // Output: - // 1 -} - -func ExampleDocumentMapping_AddFieldMapping() { - // you can only add field mapping to those properties which already have a document mapping - documentMapping := NewDocumentMapping() - subDocumentMapping := NewDocumentMapping() - documentMapping.AddSubDocumentMapping("Property", subDocumentMapping) - - fieldMapping := NewTextFieldMapping() - fieldMapping.Analyzer = "en" - subDocumentMapping.AddFieldMapping(fieldMapping) - - fmt.Println(len(documentMapping.Properties["Property"].Fields)) - // Output: - // 1 -} - -func ExampleDocumentMapping_AddFieldMappingsAt() { - // you can only add field mapping to those properties which already have a document mapping - documentMapping := NewDocumentMapping() - subDocumentMapping := NewDocumentMapping() - documentMapping.AddSubDocumentMapping("NestedProperty", subDocumentMapping) - - fieldMapping := NewTextFieldMapping() - fieldMapping.Analyzer = "en" - documentMapping.AddFieldMappingsAt("NestedProperty", fieldMapping) - - fmt.Println(len(documentMapping.Properties["NestedProperty"].Fields)) - // Output: - // 1 -} diff --git a/vendor/github.com/blevesearch/bleve/geo/README.md b/vendor/github.com/blevesearch/bleve/geo/README.md new file mode 100644 index 00000000..43bcd98f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/geo/README.md @@ -0,0 +1,9 @@ +# geo support in bleve + +First, all of this geo code is a Go adaptation of the [Lucene 5.3.2 sandbox geo support](https://lucene.apache.org/core/5_3_2/sandbox/org/apache/lucene/util/package-summary.html). + +## Notes + +- All of the APIs will use float64 for lon/lat values. +- When describing a point in function arguments or return values, we always use the order lon, lat. +- High level APIs will use TopLeft and BottomRight to describe bounding boxes. This may not map cleanly to min/max lon/lat when crossing the dateline. The lower level APIs will use min/max lon/lat and require the higher-level code to split boxes accordingly. diff --git a/vendor/github.com/blevesearch/bleve/geo/geo.go b/vendor/github.com/blevesearch/bleve/geo/geo.go new file mode 100644 index 00000000..3212539b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/geo/geo.go @@ -0,0 +1,205 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "math" + + "github.com/blevesearch/bleve/numeric" +) + +// GeoBits is the number of bits used for a single geo point +// Currently this is 32bits for lon and 32bits for lat +var GeoBits uint = 32 + +var minLon = -180.0 +var minLat = -90.0 +var geoTolerance = 1E-6 +var lonScale = float64((uint64(0x1)<> 1)) +} + +func unscaleLon(lon uint64) float64 { + return (float64(lon) / lonScale) + minLon +} + +func unscaleLat(lat uint64) float64 { + return (float64(lat) / latScale) + minLat +} + +// compareGeo will compare two float values and see if they are the same +// taking into consideration a known geo tolerance. +func compareGeo(a, b float64) float64 { + compare := a - b + if math.Abs(compare) <= geoTolerance { + return 0 + } + return compare +} + +// RectIntersects checks whether rectangles a and b intersect +func RectIntersects(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool { + return !(aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY) +} + +// RectWithin checks whether box a is within box b +func RectWithin(aMinX, aMinY, aMaxX, aMaxY, bMinX, bMinY, bMaxX, bMaxY float64) bool { + rv := !(aMinX < bMinX || aMinY < bMinY || aMaxX > bMaxX || aMaxY > bMaxY) + return rv +} + +// BoundingBoxContains checks whether the lon/lat point is within the box +func BoundingBoxContains(lon, lat, minLon, minLat, maxLon, maxLat float64) bool { + return compareGeo(lon, minLon) >= 0 && compareGeo(lon, maxLon) <= 0 && + compareGeo(lat, minLat) >= 0 && compareGeo(lat, maxLat) <= 0 +} + +// ComputeBoundingBox will compute a bounding box around the provided point +// which surrounds a circle of the provided radius (in meters). +func ComputeBoundingBox(centerLon, centerLat, + radius float64) (upperLeftLon float64, upperLeftLat float64, + lowerRightLon float64, lowerRightLat float64) { + _, tlat := pointFromLonLatBearing(centerLon, centerLat, 0, radius) + rlon, _ := pointFromLonLatBearing(centerLon, centerLat, 90, radius) + _, blat := pointFromLonLatBearing(centerLon, centerLat, 180, radius) + llon, _ := pointFromLonLatBearing(centerLon, centerLat, 270, radius) + return normalizeLon(llon), normalizeLat(tlat), + normalizeLon(rlon), normalizeLat(blat) +} + +const degreesToRadian = math.Pi / 180 +const radiansToDegrees = 180 / math.Pi +const flattening = 1.0 / 298.257223563 +const semiMajorAxis = 6378137 +const semiMinorAxis = semiMajorAxis * (1.0 - flattening) +const semiMajorAxis2 = semiMajorAxis * semiMajorAxis +const semiMinorAxis2 = semiMinorAxis * semiMinorAxis + +// DegreesToRadians converts an angle in degrees to radians +func DegreesToRadians(d float64) float64 { + return d * degreesToRadian +} + +// RadiansToDegrees converts an angle in radians to degress +func RadiansToDegrees(r float64) float64 { + return r * radiansToDegrees +} + +// pointFromLonLatBearing starts that the provide lon,lat +// then moves in the bearing direction (in degrees) +// this move continues for the provided distance (in meters) +// The lon, lat of this destination location is returned. +func pointFromLonLatBearing(lon, lat, bearing, + dist float64) (float64, float64) { + + alpha1 := DegreesToRadians(bearing) + cosA1 := math.Cos(alpha1) + sinA1 := math.Sin(alpha1) + tanU1 := (1 - flattening) * math.Tan(DegreesToRadians(lat)) + cosU1 := 1 / math.Sqrt(1+tanU1*tanU1) + sinU1 := tanU1 * cosU1 + sig1 := math.Atan2(tanU1, cosA1) + sinAlpha := cosU1 * sinA1 + cosSqAlpha := 1 - sinAlpha*sinAlpha + uSq := cosSqAlpha * (semiMajorAxis2 - semiMinorAxis2) / semiMinorAxis2 + A := 1 + uSq/16384*(4096+uSq*(-768+uSq*(320-175*uSq))) + B := uSq / 1024 * (256 + uSq*(-128+uSq*(74-47*uSq))) + + sigma := dist / (semiMinorAxis * A) + + cos25SigmaM := math.Cos(2*sig1 + sigma) + sinSigma := math.Sin(sigma) + cosSigma := math.Cos(sigma) + deltaSigma := B * sinSigma * (cos25SigmaM + (B/4)* + (cosSigma*(-1+2*cos25SigmaM*cos25SigmaM)-(B/6)*cos25SigmaM* + (-1+4*sinSigma*sinSigma)*(-3+4*cos25SigmaM*cos25SigmaM))) + sigmaP := sigma + sigma = dist/(semiMinorAxis*A) + deltaSigma + for math.Abs(sigma-sigmaP) > 1E-12 { + cos25SigmaM = math.Cos(2*sig1 + sigma) + sinSigma = math.Sin(sigma) + cosSigma = math.Cos(sigma) + deltaSigma = B * sinSigma * (cos25SigmaM + (B/4)* + (cosSigma*(-1+2*cos25SigmaM*cos25SigmaM)-(B/6)*cos25SigmaM* + (-1+4*sinSigma*sinSigma)*(-3+4*cos25SigmaM*cos25SigmaM))) + sigmaP = sigma + sigma = dist/(semiMinorAxis*A) + deltaSigma + } + + tmp := sinU1*sinSigma - cosU1*cosSigma*cosA1 + lat2 := math.Atan2(sinU1*cosSigma+cosU1*sinSigma*cosA1, + (1-flattening)*math.Sqrt(sinAlpha*sinAlpha+tmp*tmp)) + lamda := math.Atan2(sinSigma*sinA1, cosU1*cosSigma-sinU1*sinSigma*cosA1) + c := flattening / 16 * cosSqAlpha * (4 + flattening*(4-3*cosSqAlpha)) + lam := lamda - (1-c)*flattening*sinAlpha* + (sigma+c*sinSigma*(cos25SigmaM+c*cosSigma*(-1+2*cos25SigmaM*cos25SigmaM))) + + rvlon := lon + RadiansToDegrees(lam) + rvlat := RadiansToDegrees(lat2) + + return rvlon, rvlat +} + +// normalizeLon normalizes a longitude value within the -180 to 180 range +func normalizeLon(lonDeg float64) float64 { + if lonDeg >= -180 && lonDeg <= 180 { + return lonDeg + } + + off := math.Mod(lonDeg+180, 360) + if off < 0 { + return 180 + off + } else if off == 0 && lonDeg > 0 { + return 180 + } + return -180 + off +} + +// normalizeLat normalizes a latitude value within the -90 to 90 range +func normalizeLat(latDeg float64) float64 { + if latDeg >= -90 && latDeg <= 90 { + return latDeg + } + off := math.Abs(math.Mod(latDeg+90, 360)) + if off <= 180 { + return off - 90 + } + return (360 - off) - 90 +} diff --git a/vendor/github.com/blevesearch/bleve/geo/geo_dist.go b/vendor/github.com/blevesearch/bleve/geo/geo_dist.go new file mode 100644 index 00000000..d3ae0ed9 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/geo/geo_dist.go @@ -0,0 +1,98 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +type distanceUnit struct { + conv float64 + suffixes []string +} + +var inch = distanceUnit{0.0254, []string{"in", "inch"}} +var yard = distanceUnit{0.9144, []string{"yd", "yards"}} +var feet = distanceUnit{0.3048, []string{"ft", "feet"}} +var kilom = distanceUnit{1000, []string{"km", "kilometers"}} +var nauticalm = distanceUnit{1852.0, []string{"nm", "nauticalmiles"}} +var millim = distanceUnit{0.001, []string{"mm", "millimeters"}} +var centim = distanceUnit{0.01, []string{"cm", "centimeters"}} +var miles = distanceUnit{1609.344, []string{"mi", "miles"}} +var meters = distanceUnit{1, []string{"m", "meters"}} + +var distanceUnits = []*distanceUnit{ + &inch, &yard, &feet, &kilom, &nauticalm, &millim, ¢im, &miles, &meters, +} + +// ParseDistance attempts to parse a distance string and return distance in +// meters. Example formats supported: +// "5in" "5inch" "7yd" "7yards" "9ft" "9feet" "11km" "11kilometers" +// "3nm" "3nauticalmiles" "13mm" "13millimeters" "15cm" "15centimeters" +// "17mi" "17miles" "19m" "19meters" +// If the unit cannot be determined, the entire string is parsed and the +// unit of meters is assumed. +// If the number portion cannot be parsed, 0 and the parse error are returned. +func ParseDistance(d string) (float64, error) { + for _, unit := range distanceUnits { + for _, unitSuffix := range unit.suffixes { + if strings.HasSuffix(d, unitSuffix) { + parsedNum, err := strconv.ParseFloat(d[0:len(d)-len(unitSuffix)], 64) + if err != nil { + return 0, err + } + return parsedNum * unit.conv, nil + } + } + } + // no unit matched, try assuming meters? + parsedNum, err := strconv.ParseFloat(d, 64) + if err != nil { + return 0, err + } + return parsedNum, nil +} + +// ParseDistanceUnit attempts to parse a distance unit and return the +// multiplier for converting this to meters. If the unit cannot be parsed +// then 0 and the error message is returned. +func ParseDistanceUnit(u string) (float64, error) { + for _, unit := range distanceUnits { + for _, unitSuffix := range unit.suffixes { + if u == unitSuffix { + return unit.conv, nil + } + } + } + return 0, fmt.Errorf("unknown distance unit: %s", u) +} + +// Haversin computes the distance between two points. +// This implemenation uses the sloppy math implemenations which trade off +// accuracy for performance. The distance returned is in kilometers. +func Haversin(lon1, lat1, lon2, lat2 float64) float64 { + x1 := lat1 * degreesToRadian + x2 := lat2 * degreesToRadian + h1 := 1 - cos(x1-x2) + h2 := 1 - cos((lon1-lon2)*degreesToRadian) + h := (h1 + cos(x1)*cos(x2)*h2) / 2 + avgLat := (x1 + x2) / 2 + diameter := earthDiameter(avgLat) + + return diameter * asin(math.Min(1, math.Sqrt(h))) +} diff --git a/vendor/github.com/blevesearch/bleve/geo/parse.go b/vendor/github.com/blevesearch/bleve/geo/parse.go new file mode 100644 index 00000000..04a57538 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/geo/parse.go @@ -0,0 +1,140 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "reflect" + "strings" +) + +// ExtractGeoPoint takes an arbitrary interface{} and tries it's best to +// interpret it is as geo point. Supported formats: +// Container: +// slice length 2 (GeoJSON) +// first element lon, second element lat +// map[string]interface{} +// exact keys lat and lon or lng +// struct +// w/exported fields case-insensitive match on lat and lon or lng +// struct +// satisfying Later and Loner or Lnger interfaces +// +// in all cases values must be some sort of numeric-like thing: int/uint/float +func ExtractGeoPoint(thing interface{}) (lon, lat float64, success bool) { + var foundLon, foundLat bool + + thingVal := reflect.ValueOf(thing) + thingTyp := thingVal.Type() + + // is it a slice + if thingVal.IsValid() && thingVal.Kind() == reflect.Slice { + // must be length 2 + if thingVal.Len() == 2 { + first := thingVal.Index(0) + if first.CanInterface() { + firstVal := first.Interface() + lon, foundLon = extractNumericVal(firstVal) + } + second := thingVal.Index(1) + if second.CanInterface() { + secondVal := second.Interface() + lat, foundLat = extractNumericVal(secondVal) + } + } + } + + // is it a map + if l, ok := thing.(map[string]interface{}); ok { + if lval, ok := l["lon"]; ok { + lon, foundLon = extractNumericVal(lval) + } else if lval, ok := l["lng"]; ok { + lon, foundLon = extractNumericVal(lval) + } + if lval, ok := l["lat"]; ok { + lat, foundLat = extractNumericVal(lval) + } + } + + // now try reflection on struct fields + if thingVal.IsValid() && thingVal.Kind() == reflect.Struct { + for i := 0; i < thingVal.NumField(); i++ { + fieldName := thingTyp.Field(i).Name + if strings.HasPrefix(strings.ToLower(fieldName), "lon") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lon, foundLon = extractNumericVal(fieldVal) + } + } + if strings.HasPrefix(strings.ToLower(fieldName), "lng") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lon, foundLon = extractNumericVal(fieldVal) + } + } + if strings.HasPrefix(strings.ToLower(fieldName), "lat") { + if thingVal.Field(i).CanInterface() { + fieldVal := thingVal.Field(i).Interface() + lat, foundLat = extractNumericVal(fieldVal) + } + } + } + } + + // last hope, some interfaces + // lon + if l, ok := thing.(loner); ok { + lon = l.Lon() + foundLon = true + } else if l, ok := thing.(lnger); ok { + lon = l.Lng() + foundLon = true + } + // lat + if l, ok := thing.(later); ok { + lat = l.Lat() + foundLat = true + } + + return lon, lat, foundLon && foundLat +} + +// extract numeric value (if possible) and returns a float64 +func extractNumericVal(v interface{}) (float64, bool) { + val := reflect.ValueOf(v) + typ := val.Type() + switch typ.Kind() { + case reflect.Float32, reflect.Float64: + return val.Float(), true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return float64(val.Int()), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return float64(val.Uint()), true + } + + return 0, false +} + +// various support interfaces which can be used to find lat/lon +type loner interface { + Lon() float64 +} + +type later interface { + Lat() float64 +} + +type lnger interface { + Lng() float64 +} diff --git a/vendor/github.com/blevesearch/bleve/geo/sloppy.go b/vendor/github.com/blevesearch/bleve/geo/sloppy.go new file mode 100644 index 00000000..a0f5a366 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/geo/sloppy.go @@ -0,0 +1,206 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package geo + +import ( + "math" +) + +var earthDiameterPerLatitude []float64 +var sinTab []float64 +var cosTab []float64 +var asinTab []float64 +var asinDer1DivF1Tab []float64 +var asinDer2DivF2Tab []float64 +var asinDer3DivF3Tab []float64 +var asinDer4DivF4Tab []float64 + +const radiusTabsSize = (1 << 10) + 1 +const radiusDelta = (math.Pi / 2) / (radiusTabsSize - 1) +const radiusIndexer = 1 / radiusDelta +const sinCosTabsSize = (1 << 11) + 1 +const asinTabsSize = (1 << 13) + 1 +const oneDivF2 = 1 / 2.0 +const oneDivF3 = 1 / 6.0 +const oneDivF4 = 1 / 24.0 + +// 1.57079632673412561417e+00 first 33 bits of pi/2 +var pio2Hi = math.Float64frombits(0x3FF921FB54400000) + +// 6.07710050650619224932e-11 pi/2 - PIO2_HI +var pio2Lo = math.Float64frombits(0x3DD0B4611A626331) + +var asinPio2Hi = math.Float64frombits(0x3FF921FB54442D18) // 1.57079632679489655800e+00 +var asinPio2Lo = math.Float64frombits(0x3C91A62633145C07) // 6.12323399573676603587e-17 +var asinPs0 = math.Float64frombits(0x3fc5555555555555) // 1.66666666666666657415e-01 +var asinPs1 = math.Float64frombits(0xbfd4d61203eb6f7d) // -3.25565818622400915405e-01 +var asinPs2 = math.Float64frombits(0x3fc9c1550e884455) // 2.01212532134862925881e-01 +var asinPs3 = math.Float64frombits(0xbfa48228b5688f3b) // -4.00555345006794114027e-02 +var asinPs4 = math.Float64frombits(0x3f49efe07501b288) // 7.91534994289814532176e-04 +var asinPs5 = math.Float64frombits(0x3f023de10dfdf709) // 3.47933107596021167570e-05 +var asinQs1 = math.Float64frombits(0xc0033a271c8a2d4b) // -2.40339491173441421878e+00 +var asinQs2 = math.Float64frombits(0x40002ae59c598ac8) // 2.02094576023350569471e+00 +var asinQs3 = math.Float64frombits(0xbfe6066c1b8d0159) // -6.88283971605453293030e-01 +var asinQs4 = math.Float64frombits(0x3fb3b8c5b12e9282) // 7.70381505559019352791e-02 + +var twoPiHi = 4 * pio2Hi +var twoPiLo = 4 * pio2Lo +var sinCosDeltaHi = twoPiHi/sinCosTabsSize - 1 +var sinCosDeltaLo = twoPiLo/sinCosTabsSize - 1 +var sinCosIndexer = 1 / (sinCosDeltaHi + sinCosDeltaLo) +var sinCosMaxValueForIntModulo = ((math.MaxInt64 >> 9) / sinCosIndexer) * 0.99 +var asinMaxValueForTabs = math.Sin(73.0 * degreesToRadian) + +var asinDelta = asinMaxValueForTabs / (asinTabsSize - 1) +var asinIndexer = 1 / asinDelta + +func init() { + // initializes the tables used for the sloppy math functions + + // sin and cos + sinTab = make([]float64, sinCosTabsSize) + cosTab = make([]float64, sinCosTabsSize) + sinCosPiIndex := (sinCosTabsSize - 1) / 2 + sinCosPiMul2Index := 2 * sinCosPiIndex + sinCosPiMul05Index := sinCosPiIndex / 2 + sinCosPiMul15Index := 3 * sinCosPiIndex / 2 + for i := 0; i < sinCosTabsSize; i++ { + // angle: in [0,2*PI]. + angle := float64(i)*sinCosDeltaHi + float64(i)*sinCosDeltaLo + sinAngle := math.Sin(angle) + cosAngle := math.Cos(angle) + // For indexes corresponding to null cosine or sine, we make sure the value is zero + // and not an epsilon. This allows for a much better accuracy for results close to zero. + if i == sinCosPiIndex { + sinAngle = 0.0 + } else if i == sinCosPiMul2Index { + sinAngle = 0.0 + } else if i == sinCosPiMul05Index { + sinAngle = 0.0 + } else if i == sinCosPiMul15Index { + sinAngle = 0.0 + } + sinTab[i] = sinAngle + cosTab[i] = cosAngle + } + + // asin + asinTab = make([]float64, asinTabsSize) + asinDer1DivF1Tab = make([]float64, asinTabsSize) + asinDer2DivF2Tab = make([]float64, asinTabsSize) + asinDer3DivF3Tab = make([]float64, asinTabsSize) + asinDer4DivF4Tab = make([]float64, asinTabsSize) + for i := 0; i < asinTabsSize; i++ { + // x: in [0,ASIN_MAX_VALUE_FOR_TABS]. + x := float64(i) * asinDelta + asinTab[i] = math.Asin(x) + oneMinusXSqInv := 1.0 / (1 - x*x) + oneMinusXSqInv05 := math.Sqrt(oneMinusXSqInv) + oneMinusXSqInv15 := oneMinusXSqInv05 * oneMinusXSqInv + oneMinusXSqInv25 := oneMinusXSqInv15 * oneMinusXSqInv + oneMinusXSqInv35 := oneMinusXSqInv25 * oneMinusXSqInv + asinDer1DivF1Tab[i] = oneMinusXSqInv05 + asinDer2DivF2Tab[i] = (x * oneMinusXSqInv15) * oneDivF2 + asinDer3DivF3Tab[i] = ((1 + 2*x*x) * oneMinusXSqInv25) * oneDivF3 + asinDer4DivF4Tab[i] = ((5 + 2*x*(2+x*(5-2*x))) * oneMinusXSqInv35) * oneDivF4 + } + + // earth radius + a := 6378137.0 + b := 6356752.31420 + a2 := a * a + b2 := b * b + earthDiameterPerLatitude = make([]float64, radiusTabsSize) + earthDiameterPerLatitude[0] = 2.0 * a / 1000 + earthDiameterPerLatitude[radiusTabsSize-1] = 2.0 * b / 1000 + for i := 1; i < radiusTabsSize-1; i++ { + lat := math.Pi * float64(i) / (2*radiusTabsSize - 1) + one := math.Pow(a2*math.Cos(lat), 2) + two := math.Pow(b2*math.Sin(lat), 2) + three := math.Pow(float64(a)*math.Cos(lat), 2) + four := math.Pow(b*math.Sin(lat), 2) + radius := math.Sqrt((one + two) / (three + four)) + earthDiameterPerLatitude[i] = 2 * radius / 1000 + } +} + +// earthDiameter returns an estimation of the earth's diameter at the specified +// latitude in kilometers +func earthDiameter(lat float64) float64 { + index := math.Mod(math.Abs(lat)*radiusIndexer+0.5, float64(len(earthDiameterPerLatitude))) + if math.IsNaN(index) { + return 0 + } + return earthDiameterPerLatitude[int(index)] +} + +// cos is a sloppy math (faster) implementation of math.Cos +func cos(a float64) float64 { + if a < 0.0 { + a = -a + } + if a > sinCosMaxValueForIntModulo { + return math.Cos(a) + } + // index: possibly outside tables range. + index := int(a*sinCosIndexer + 0.5) + delta := (a - float64(index)*sinCosDeltaHi) - float64(index)*sinCosDeltaLo + // Making sure index is within tables range. + // Last value of each table is the same than first, so we ignore it (tabs size minus one) for modulo. + index &= (sinCosTabsSize - 2) // index % (SIN_COS_TABS_SIZE-1) + indexCos := cosTab[index] + indexSin := sinTab[index] + return indexCos + delta*(-indexSin+delta*(-indexCos*oneDivF2+delta*(indexSin*oneDivF3+delta*indexCos*oneDivF4))) +} + +// asin is a sloppy math (faster) implementation of math.Asin +func asin(a float64) float64 { + var negateResult bool + if a < 0 { + a = -a + negateResult = true + } + if a <= asinMaxValueForTabs { + index := int(a*asinIndexer + 0.5) + delta := a - float64(index)*asinDelta + result := asinTab[index] + delta*(asinDer1DivF1Tab[index]+delta*(asinDer2DivF2Tab[index]+delta*(asinDer3DivF3Tab[index]+delta*asinDer4DivF4Tab[index]))) + if negateResult { + return -result + } + return result + } + // value > ASIN_MAX_VALUE_FOR_TABS, or value is NaN + // This part is derived from fdlibm. + if a < 1 { + t := (1.0 - a) * 0.5 + p := t * (asinPs0 + t*(asinPs1+t*(asinPs2+t*(asinPs3+t*(asinPs4+t+asinPs5))))) + q := 1.0 + t*(asinQs1+t*(asinQs2+t*(asinQs3+t*asinQs4))) + s := math.Sqrt(t) + z := s + s*(p/q) + result := asinPio2Hi - ((z + z) - asinPio2Lo) + if negateResult { + return -result + } + return result + } + // value >= 1.0, or value is NaN + if a == 1.0 { + if negateResult { + return -math.Pi / 2 + } + return math.Pi / 2 + } + return math.NaN() +} diff --git a/vendor/github.com/blevesearch/bleve/http/alias.go b/vendor/github.com/blevesearch/bleve/http/alias.go deleted file mode 100644 index 4bd5d826..00000000 --- a/vendor/github.com/blevesearch/bleve/http/alias.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" -) - -type AliasAction struct { - Alias string `json:"alias"` - AddIndexes []string `json:"add"` - RemoveIndexes []string `json:"remove"` -} - -type AliasHandler struct{} - -func NewAliasHandler() *AliasHandler { - return &AliasHandler{} -} - -func (h *AliasHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - - // read the request body - requestBody, err := ioutil.ReadAll(req.Body) - if err != nil { - showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) - return - } - - var aliasAction AliasAction - // interpret request body as alias actions - if len(requestBody) > 0 { - err := json.Unmarshal(requestBody, &aliasAction) - if err != nil { - showError(w, req, fmt.Sprintf("error parsing alias actions: %v", err), 400) - return - } - } else { - showError(w, req, "request body must contain alias actions", 400) - return - } - - err = UpdateAlias(aliasAction.Alias, aliasAction.AddIndexes, aliasAction.RemoveIndexes) - if err != nil { - showError(w, req, fmt.Sprintf("error updating alias: %v", err), 400) - return - } - - rv := struct { - Status string `json:"status"` - }{ - Status: "ok", - } - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/debug.go b/vendor/github.com/blevesearch/bleve/http/debug.go deleted file mode 100644 index 4e88cd24..00000000 --- a/vendor/github.com/blevesearch/bleve/http/debug.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "net/http" - - "github.com/blevesearch/bleve/index/upside_down" -) - -// DebugDocumentHandler allows you to debug the index content -// for a given document id. -type DebugDocumentHandler struct { - defaultIndexName string - IndexNameLookup varLookupFunc - DocIDLookup varLookupFunc -} - -func NewDebugDocumentHandler(defaultIndexName string) *DebugDocumentHandler { - return &DebugDocumentHandler{ - defaultIndexName: defaultIndexName, - } -} - -func (h *DebugDocumentHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - - // find the index to operate on - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - indexName = h.defaultIndexName - } - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - // find the docID - var docID string - if h.DocIDLookup != nil { - docID = h.DocIDLookup(req) - } - - rv := make([]interface{}, 0) - rowChan := index.DumpDoc(docID) - for row := range rowChan { - switch row := row.(type) { - case error: - showError(w, req, fmt.Sprintf("error debugging document: %v", row), 500) - return - case upside_down.UpsideDownCouchRow: - tmp := struct { - Key []byte `json:"key"` - Val []byte `json:"val"` - }{ - Key: row.Key(), - Val: row.Value(), - } - rv = append(rv, tmp) - } - } - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/doc_count.go b/vendor/github.com/blevesearch/bleve/http/doc_count.go deleted file mode 100644 index 2318190c..00000000 --- a/vendor/github.com/blevesearch/bleve/http/doc_count.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "net/http" -) - -type DocCountHandler struct { - defaultIndexName string - IndexNameLookup varLookupFunc -} - -func NewDocCountHandler(defaultIndexName string) *DocCountHandler { - return &DocCountHandler{ - defaultIndexName: defaultIndexName, - } -} - -func (h *DocCountHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - // find the index to operate on - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - indexName = h.defaultIndexName - } - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - docCount, err := index.DocCount() - if err != nil { - showError(w, req, fmt.Sprintf("error counting docs: %v", err), 500) - return - } - rv := struct { - Status string `json:"status"` - Count uint64 `json:"count"` - }{ - Status: "ok", - Count: docCount, - } - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/doc_delete.go b/vendor/github.com/blevesearch/bleve/http/doc_delete.go deleted file mode 100644 index 0c136c21..00000000 --- a/vendor/github.com/blevesearch/bleve/http/doc_delete.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "net/http" -) - -type DocDeleteHandler struct { - defaultIndexName string - IndexNameLookup varLookupFunc - DocIDLookup varLookupFunc -} - -func NewDocDeleteHandler(defaultIndexName string) *DocDeleteHandler { - return &DocDeleteHandler{ - defaultIndexName: defaultIndexName, - } -} - -func (h *DocDeleteHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - - // find the index to operate on - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - indexName = h.defaultIndexName - } - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - // find the doc id - var docID string - if h.DocIDLookup != nil { - docID = h.DocIDLookup(req) - } - if docID == "" { - showError(w, req, "document id cannot be empty", 400) - return - } - - err := index.Delete(docID) - if err != nil { - showError(w, req, fmt.Sprintf("error deleting document '%s': %v", docID, err), 500) - return - } - - rv := struct { - Status string `json:"status"` - }{ - Status: "ok", - } - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/doc_get.go b/vendor/github.com/blevesearch/bleve/http/doc_get.go deleted file mode 100644 index 0917cd60..00000000 --- a/vendor/github.com/blevesearch/bleve/http/doc_get.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "net/http" - "time" - - "github.com/blevesearch/bleve/document" -) - -type DocGetHandler struct { - defaultIndexName string - IndexNameLookup varLookupFunc - DocIDLookup varLookupFunc -} - -func NewDocGetHandler(defaultIndexName string) *DocGetHandler { - return &DocGetHandler{ - defaultIndexName: defaultIndexName, - } -} - -func (h *DocGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - // find the index to operate on - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - indexName = h.defaultIndexName - } - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - // find the doc id - var docID string - if h.DocIDLookup != nil { - docID = h.DocIDLookup(req) - } - if docID == "" { - showError(w, req, "document id cannot be empty", 400) - return - } - - doc, err := index.Document(docID) - if err != nil { - showError(w, req, fmt.Sprintf("error deleting document '%s': %v", docID, err), 500) - return - } - if doc == nil { - showError(w, req, fmt.Sprintf("no such document '%s'", docID), 404) - return - } - - rv := struct { - ID string `json:"id"` - Fields map[string]interface{} `json:"fields"` - }{ - ID: docID, - Fields: map[string]interface{}{}, - } - for _, field := range doc.Fields { - var newval interface{} - switch field := field.(type) { - case *document.TextField: - newval = string(field.Value()) - case *document.NumericField: - n, err := field.Number() - if err == nil { - newval = n - } - case *document.DateTimeField: - d, err := field.DateTime() - if err == nil { - newval = d.Format(time.RFC3339Nano) - } - } - existing, existed := rv.Fields[field.Name()] - if existed { - switch existing := existing.(type) { - case []interface{}: - rv.Fields[field.Name()] = append(existing, newval) - case interface{}: - arr := make([]interface{}, 2) - arr[0] = existing - arr[1] = newval - rv.Fields[field.Name()] = arr - } - } else { - rv.Fields[field.Name()] = newval - } - } - - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/doc_index.go b/vendor/github.com/blevesearch/bleve/http/doc_index.go deleted file mode 100644 index d076417f..00000000 --- a/vendor/github.com/blevesearch/bleve/http/doc_index.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "io/ioutil" - "net/http" -) - -type DocIndexHandler struct { - defaultIndexName string - IndexNameLookup varLookupFunc - DocIDLookup varLookupFunc -} - -func NewDocIndexHandler(defaultIndexName string) *DocIndexHandler { - return &DocIndexHandler{ - defaultIndexName: defaultIndexName, - } -} - -func (h *DocIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - - // find the index to operate on - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - indexName = h.defaultIndexName - } - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - // find the doc id - var docID string - if h.DocIDLookup != nil { - docID = h.DocIDLookup(req) - } - if docID == "" { - showError(w, req, "document id cannot be empty", 400) - return - } - - // read the request body - requestBody, err := ioutil.ReadAll(req.Body) - if err != nil { - showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) - return - } - - err = index.Index(docID, requestBody) - if err != nil { - showError(w, req, fmt.Sprintf("error indexing document '%s': %v", docID, err), 500) - return - } - - rv := struct { - Status string `json:"status"` - }{ - Status: "ok", - } - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/fields.go b/vendor/github.com/blevesearch/bleve/http/fields.go deleted file mode 100644 index dd3afba2..00000000 --- a/vendor/github.com/blevesearch/bleve/http/fields.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "net/http" -) - -type ListFieldsHandler struct { - defaultIndexName string - IndexNameLookup varLookupFunc -} - -func NewListFieldsHandler(defaultIndexName string) *ListFieldsHandler { - return &ListFieldsHandler{ - defaultIndexName: defaultIndexName, - } -} - -func (h *ListFieldsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - - // find the index to operate on - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - indexName = h.defaultIndexName - } - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - fields, err := index.Fields() - if err != nil { - showError(w, req, fmt.Sprintf("error: %v", err), 500) - return - } - - fieldsResponse := struct { - Fields []string `json:"fields"` - }{ - Fields: fields, - } - - // encode the response - mustEncode(w, fieldsResponse) -} diff --git a/vendor/github.com/blevesearch/bleve/http/handlers_test.go b/vendor/github.com/blevesearch/bleve/http/handlers_test.go deleted file mode 100644 index 492f9bb5..00000000 --- a/vendor/github.com/blevesearch/bleve/http/handlers_test.go +++ /dev/null @@ -1,704 +0,0 @@ -package http - -import ( - "bytes" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "os" - "reflect" - "testing" -) - -func docIDLookup(req *http.Request) string { - return req.FormValue("docID") -} - -func indexNameLookup(req *http.Request) string { - return req.FormValue("indexName") -} - -func TestHandlers(t *testing.T) { - - basePath := "testbase" - err := os.MkdirAll(basePath, 0700) - if err != nil { - t.Fatal(err) - } - defer func() { - err := os.RemoveAll(basePath) - if err != nil { - t.Fatal(err) - } - }() - - createIndexHandler := NewCreateIndexHandler(basePath) - createIndexHandler.IndexNameLookup = indexNameLookup - - getIndexHandler := NewGetIndexHandler() - getIndexHandler.IndexNameLookup = indexNameLookup - - deleteIndexHandler := NewDeleteIndexHandler(basePath) - deleteIndexHandler.IndexNameLookup = indexNameLookup - - listIndexesHandler := NewListIndexesHandler() - - docIndexHandler := NewDocIndexHandler("") - docIndexHandler.IndexNameLookup = indexNameLookup - docIndexHandler.DocIDLookup = docIDLookup - - docCountHandler := NewDocCountHandler("") - docCountHandler.IndexNameLookup = indexNameLookup - - docGetHandler := NewDocGetHandler("") - docGetHandler.IndexNameLookup = indexNameLookup - docGetHandler.DocIDLookup = docIDLookup - - docDeleteHandler := NewDocDeleteHandler("") - docDeleteHandler.IndexNameLookup = indexNameLookup - docDeleteHandler.DocIDLookup = docIDLookup - - searchHandler := NewSearchHandler("") - searchHandler.IndexNameLookup = indexNameLookup - - listFieldsHandler := NewListFieldsHandler("") - listFieldsHandler.IndexNameLookup = indexNameLookup - - debugHandler := NewDebugDocumentHandler("") - debugHandler.IndexNameLookup = indexNameLookup - debugHandler.DocIDLookup = docIDLookup - - aliasHandler := NewAliasHandler() - - tests := []struct { - Desc string - Handler http.Handler - Path string - Method string - Params url.Values - Body []byte - Status int - ResponseBody []byte - ResponseMatch map[string]bool - }{ - { - Desc: "create index", - Handler: createIndexHandler, - Path: "/create", - Method: "PUT", - Params: url.Values{"indexName": []string{"ti1"}}, - Body: []byte("{}"), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "create existing index", - Handler: createIndexHandler, - Path: "/create", - Method: "PUT", - Params: url.Values{"indexName": []string{"ti1"}}, - Body: []byte("{}"), - Status: http.StatusInternalServerError, - ResponseMatch: map[string]bool{ - `path already exists`: true, - }, - }, - { - Desc: "create index missing index", - Handler: createIndexHandler, - Path: "/create", - Method: "PUT", - Body: []byte("{}"), - Status: http.StatusBadRequest, - ResponseBody: []byte(`index name is required`), - }, - { - Desc: "create index invalid json", - Handler: createIndexHandler, - Path: "/create", - Method: "PUT", - Params: url.Values{"indexName": []string{"ti9"}}, - Body: []byte("{"), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `error parsing index mapping`: true, - }, - }, - { - Desc: "get index", - Handler: getIndexHandler, - Path: "/get", - Method: "GET", - Params: url.Values{"indexName": []string{"ti1"}}, - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"status":"ok"`: true, - `"name":"ti1"`: true, - }, - }, - { - Desc: "get index does not exist", - Handler: getIndexHandler, - Path: "/get", - Method: "GET", - Params: url.Values{"indexName": []string{"dne"}}, - Status: http.StatusNotFound, - ResponseMatch: map[string]bool{ - `no such index`: true, - }, - }, - { - Desc: "get index missing name", - Handler: getIndexHandler, - Path: "/get", - Method: "GET", - Status: http.StatusBadRequest, - ResponseBody: []byte(`index name is required`), - }, - { - Desc: "create another index", - Handler: createIndexHandler, - Path: "/create", - Method: "PUT", - Params: url.Values{"indexName": []string{"ti2"}}, - Body: []byte("{}"), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "list indexes", - Handler: listIndexesHandler, - Path: "/list", - Method: "GET", - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"status":"ok"`: true, - `"ti1"`: true, - `"ti2"`: true, - }, - }, - { - Desc: "delete index", - Handler: deleteIndexHandler, - Path: "/delete", - Method: "DELETE", - Params: url.Values{"indexName": []string{"ti2"}}, - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "delete index missing name", - Handler: deleteIndexHandler, - Path: "/delete", - Method: "DELETE", - Status: http.StatusBadRequest, - ResponseBody: []byte(`index name is required`), - }, - { - Desc: "list indexes after delete", - Handler: listIndexesHandler, - Path: "/list", - Method: "GET", - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"status":"ok"`: true, - `"ti1"`: true, - `"ti2"`: false, - }, - }, - { - Desc: "index doc", - Handler: docIndexHandler, - Path: "/ti1/a", - Method: "PUT", - Params: url.Values{ - "indexName": []string{"ti1"}, - "docID": []string{"a"}, - }, - Body: []byte(`{"name":"a","body":"test","rating":7,"created":"2014-11-26","former_ratings":[3,4,2]}`), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "index doc invalid index", - Handler: docIndexHandler, - Path: "/tix/a", - Method: "PUT", - Params: url.Values{ - "indexName": []string{"tix"}, - "docID": []string{"a"}, - }, - Body: []byte(`{"name":"a","body":"test","rating":7,"created":"2014-11-26","former_ratings":[3,4,2]}`), - Status: http.StatusNotFound, - ResponseBody: []byte(`no such index 'tix'`), - }, - { - Desc: "index doc missing ID", - Handler: docIndexHandler, - Path: "/ti1/a", - Method: "PUT", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Body: []byte(`{"name":"a","body":"test","rating":7,"created":"2014-11-26","former_ratings":[3,4,2]}`), - Status: http.StatusBadRequest, - ResponseBody: []byte(`document id cannot be empty`), - }, - { - Desc: "doc count", - Handler: docCountHandler, - Path: "/ti1/count", - Method: "GET", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok","count":1}`), - }, - { - Desc: "doc count invalid index", - Handler: docCountHandler, - Path: "/tix/count", - Method: "GET", - Params: url.Values{ - "indexName": []string{"tix"}, - }, - Status: http.StatusNotFound, - ResponseBody: []byte(`no such index 'tix'`), - }, - { - Desc: "doc get", - Handler: docGetHandler, - Path: "/ti1/a", - Method: "GET", - Params: url.Values{ - "indexName": []string{"ti1"}, - "docID": []string{"a"}, - }, - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"id":"a"`: true, - `"body":"test"`: true, - `"name":"a"`: true, - }, - }, - { - Desc: "doc get invalid index", - Handler: docGetHandler, - Path: "/tix/a", - Method: "GET", - Params: url.Values{ - "indexName": []string{"tix"}, - "docID": []string{"a"}, - }, - Status: http.StatusNotFound, - ResponseBody: []byte(`no such index 'tix'`), - }, - { - Desc: "doc get missing ID", - Handler: docGetHandler, - Path: "/ti1/a", - Method: "GET", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Status: http.StatusBadRequest, - ResponseBody: []byte(`document id cannot be empty`), - }, - { - Desc: "index another doc", - Handler: docIndexHandler, - Path: "/ti1/b", - Method: "PUT", - Params: url.Values{ - "indexName": []string{"ti1"}, - "docID": []string{"b"}, - }, - Body: []byte(`{"name":"b","body":"del"}`), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "doc count again", - Handler: docCountHandler, - Path: "/ti1/count", - Method: "GET", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok","count":2}`), - }, - { - Desc: "delete doc", - Handler: docDeleteHandler, - Path: "/ti1/b", - Method: "DELETE", - Params: url.Values{ - "indexName": []string{"ti1"}, - "docID": []string{"b"}, - }, - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "delete doc invalid index", - Handler: docDeleteHandler, - Path: "/tix/b", - Method: "DELETE", - Params: url.Values{ - "indexName": []string{"tix"}, - "docID": []string{"b"}, - }, - Status: http.StatusNotFound, - ResponseBody: []byte(`no such index 'tix'`), - }, - { - Desc: "delete doc missing docID", - Handler: docDeleteHandler, - Path: "/ti1/b", - Method: "DELETE", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Status: http.StatusBadRequest, - ResponseBody: []byte(`document id cannot be empty`), - }, - { - Desc: "doc get", - Handler: docGetHandler, - Path: "/ti1/b", - Method: "GET", - Params: url.Values{ - "indexName": []string{"ti1"}, - "docID": []string{"b"}, - }, - Status: http.StatusNotFound, - ResponseMatch: map[string]bool{ - `no such document`: true, - }, - }, - { - Desc: "search", - Handler: searchHandler, - Path: "/ti1/search", - Method: "POST", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Body: []byte(`{ - "from": 0, - "size": 10, - "query": { - "fuzziness": 0, - "prefix_length": 0, - "field": "body", - "match": "test" - } - }`), - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"total_hits":1`: true, - `"id":"a"`: true, - }, - }, - { - Desc: "search index doesnt exist", - Handler: searchHandler, - Path: "/tix/search", - Method: "POST", - Params: url.Values{ - "indexName": []string{"tix"}, - }, - Body: []byte(`{ - "from": 0, - "size": 10, - "query": { - "fuzziness": 0, - "prefix_length": 0, - "field": "body", - "match": "test" - } - }`), - Status: http.StatusNotFound, - ResponseBody: []byte(`no such index 'tix'`), - }, - { - Desc: "search invalid json", - Handler: searchHandler, - Path: "/ti1/search", - Method: "POST", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Body: []byte(`{`), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `error parsing query`: true, - }, - }, - { - Desc: "search query does not validate", - Handler: searchHandler, - Path: "/ti1/search", - Method: "POST", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Body: []byte(`{ - "from": 0, - "size": 10, - "query": { - "field": "body", - "terms": [] - } - }`), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `error validating query`: true, - }, - }, - { - Desc: "list fields", - Handler: listFieldsHandler, - Path: "/ti1/fields", - Method: "GET", - Params: url.Values{ - "indexName": []string{"ti1"}, - }, - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"fields":`: true, - `"name"`: true, - `"body"`: true, - `"_all"`: true, - }, - }, - { - Desc: "list fields invalid index", - Handler: listFieldsHandler, - Path: "/tix/fields", - Method: "GET", - Params: url.Values{ - "indexName": []string{"tix"}, - }, - Status: http.StatusNotFound, - ResponseBody: []byte(`no such index 'tix'`), - }, - { - Desc: "debug doc", - Handler: debugHandler, - Path: "/ti1/a/debug", - Method: "GET", - Params: url.Values{ - "indexName": []string{"ti1"}, - "docID": []string{"a"}, - }, - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"key"`: true, - `"val"`: true, - }, - }, - { - Desc: "debug doc invalid index", - Handler: debugHandler, - Path: "/ti1/a/debug", - Method: "GET", - Params: url.Values{ - "indexName": []string{"tix"}, - "docID": []string{"a"}, - }, - Status: http.StatusNotFound, - ResponseBody: []byte(`no such index 'tix'`), - }, - { - Desc: "create alias", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "a1", - "add": ["ti1"] - }`), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "create alias invalid json", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{`), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `error parsing alias actions`: true, - }, - }, - { - Desc: "create alias empty", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(``), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `request body must contain alias actions`: true, - }, - }, - { - Desc: "create alias referring to non-existant index", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "a2", - "add": ["tix"] - }`), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `index named 'tix' does not exist`: true, - }, - }, - { - Desc: "create alias removing from new", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "a2", - "remove": ["ti1"] - }`), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `cannot remove indexes from a new alias`: true, - }, - }, - { - Desc: "create alias same name as index", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "ti1", - "remove": ["ti1"] - }`), - Status: http.StatusBadRequest, - ResponseMatch: map[string]bool{ - `is not an alias`: true, - }, - }, - { - Desc: "search alias", - Handler: searchHandler, - Path: "/a1/search", - Method: "POST", - Params: url.Values{ - "indexName": []string{"a1"}, - }, - Body: []byte(`{ - "from": 0, - "size": 10, - "query": { - "fuzziness": 0, - "prefix_length": 0, - "field": "body", - "match": "test" - } - }`), - Status: http.StatusOK, - ResponseMatch: map[string]bool{ - `"total_hits":1`: true, - `"id":"a"`: true, - }, - }, - { - Desc: "create index to add to alias", - Handler: createIndexHandler, - Path: "/create", - Method: "PUT", - Params: url.Values{"indexName": []string{"ti6"}}, - Body: []byte("{}"), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "update alias add ti6", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "a1", - "add": ["ti6"] - }`), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "update alias add doesnt exist", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "a1", - "add": ["ti99"] - }`), - Status: http.StatusBadRequest, - ResponseBody: []byte(`error updating alias: index named 'ti99' does not exist`), - }, - { - Desc: "update alias remove ti6", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "a1", - "remove": ["ti6"] - }`), - Status: http.StatusOK, - ResponseBody: []byte(`{"status":"ok"}`), - }, - { - Desc: "update alias remove doesnt exist", - Handler: aliasHandler, - Path: "/alias", - Method: "POST", - Body: []byte(`{ - "alias": "a1", - "remove": ["ti98"] - }`), - Status: http.StatusBadRequest, - ResponseBody: []byte(`error updating alias: index named 'ti98' does not exist`), - }, - } - - for _, test := range tests { - record := httptest.NewRecorder() - req := &http.Request{ - Method: test.Method, - URL: &url.URL{Path: test.Path}, - Form: test.Params, - Body: ioutil.NopCloser(bytes.NewBuffer(test.Body)), - } - test.Handler.ServeHTTP(record, req) - if got, want := record.Code, test.Status; got != want { - t.Errorf("%s: response code = %d, want %d", test.Desc, got, want) - t.Errorf("%s: response body = %s", test.Desc, record.Body) - } - - got := bytes.TrimRight(record.Body.Bytes(), "\n") - if test.ResponseBody != nil { - if !reflect.DeepEqual(got, test.ResponseBody) { - t.Errorf("%s: expected: '%s', got: '%s'", test.Desc, test.ResponseBody, got) - } - } - for pattern, shouldMatch := range test.ResponseMatch { - didMatch := bytes.Contains(got, []byte(pattern)) - if didMatch != shouldMatch { - t.Errorf("%s: expected match %t for pattern %s, got %t", test.Desc, shouldMatch, pattern, didMatch) - t.Errorf("%s: response body was: %s", test.Desc, got) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/http/index_create.go b/vendor/github.com/blevesearch/bleve/http/index_create.go deleted file mode 100644 index 532cd5c2..00000000 --- a/vendor/github.com/blevesearch/bleve/http/index_create.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - - "github.com/blevesearch/bleve" -) - -type CreateIndexHandler struct { - basePath string - IndexNameLookup varLookupFunc -} - -func NewCreateIndexHandler(basePath string) *CreateIndexHandler { - return &CreateIndexHandler{ - basePath: basePath, - } -} - -func (h *CreateIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - // find the name of the index to create - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - showError(w, req, "index name is required", 400) - return - } - - indexMapping := bleve.NewIndexMapping() - - // read the request body - requestBody, err := ioutil.ReadAll(req.Body) - if err != nil { - showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) - return - } - - // interpret request body as index mapping - if len(requestBody) > 0 { - err := json.Unmarshal(requestBody, &indexMapping) - if err != nil { - showError(w, req, fmt.Sprintf("error parsing index mapping: %v", err), 400) - return - } - } - - newIndex, err := bleve.New(h.indexPath(indexName), indexMapping) - if err != nil { - showError(w, req, fmt.Sprintf("error creating index: %v", err), 500) - return - } - RegisterIndexName(indexName, newIndex) - rv := struct { - Status string `json:"status"` - }{ - Status: "ok", - } - mustEncode(w, rv) -} - -func (h *CreateIndexHandler) indexPath(name string) string { - return h.basePath + string(os.PathSeparator) + name -} diff --git a/vendor/github.com/blevesearch/bleve/http/index_delete.go b/vendor/github.com/blevesearch/bleve/http/index_delete.go deleted file mode 100644 index 43171de4..00000000 --- a/vendor/github.com/blevesearch/bleve/http/index_delete.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "net/http" - "os" -) - -type DeleteIndexHandler struct { - basePath string - IndexNameLookup varLookupFunc -} - -func NewDeleteIndexHandler(basePath string) *DeleteIndexHandler { - return &DeleteIndexHandler{ - basePath: basePath, - } -} - -func (h *DeleteIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - // find the name of the index to delete - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - showError(w, req, "index name is required", 400) - return - } - - indexToDelete := UnregisterIndexByName(indexName) - if indexToDelete == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - // close the index - err := indexToDelete.Close() - if err != nil { - showError(w, req, fmt.Sprintf("error closing index: %v", err), 500) - return - } - - // now delete it - err = os.RemoveAll(h.indexPath(indexName)) - if err != nil { - showError(w, req, fmt.Sprintf("error deleting index: %v", err), 500) - return - } - - rv := struct { - Status string `json:"status"` - }{ - Status: "ok", - } - mustEncode(w, rv) -} - -func (h *DeleteIndexHandler) indexPath(name string) string { - return h.basePath + string(os.PathSeparator) + name -} diff --git a/vendor/github.com/blevesearch/bleve/http/index_get.go b/vendor/github.com/blevesearch/bleve/http/index_get.go deleted file mode 100644 index 360e2cd6..00000000 --- a/vendor/github.com/blevesearch/bleve/http/index_get.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "net/http" - - "github.com/blevesearch/bleve" -) - -type GetIndexHandler struct { - IndexNameLookup varLookupFunc -} - -func NewGetIndexHandler() *GetIndexHandler { - return &GetIndexHandler{} -} - -func (h *GetIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - // find the name of the index to create - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - showError(w, req, "index name is required", 400) - return - } - - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - rv := struct { - Status string `json:"status"` - Name string `json:"name"` - Mapping *bleve.IndexMapping `json:"mapping"` - }{ - Status: "ok", - Name: indexName, - Mapping: index.Mapping(), - } - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/index_list.go b/vendor/github.com/blevesearch/bleve/http/index_list.go deleted file mode 100644 index ffd09f76..00000000 --- a/vendor/github.com/blevesearch/bleve/http/index_list.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "net/http" -) - -type ListIndexesHandler struct { -} - -func NewListIndexesHandler() *ListIndexesHandler { - return &ListIndexesHandler{} -} - -func (h *ListIndexesHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - indexNames := IndexNames() - rv := struct { - Status string `json:"status"` - Indexes []string `json:"indexes"` - }{ - Status: "ok", - Indexes: indexNames, - } - mustEncode(w, rv) -} diff --git a/vendor/github.com/blevesearch/bleve/http/registry.go b/vendor/github.com/blevesearch/bleve/http/registry.go deleted file mode 100644 index 35d22804..00000000 --- a/vendor/github.com/blevesearch/bleve/http/registry.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "fmt" - "sync" - - "github.com/blevesearch/bleve" -) - -var indexNameMapping map[string]bleve.Index -var indexNameMappingLock sync.RWMutex -var indexStats = bleve.IndexStats{} - -func RegisterIndexName(name string, idx bleve.Index) { - indexNameMappingLock.Lock() - defer indexNameMappingLock.Unlock() - - if indexNameMapping == nil { - indexNameMapping = make(map[string]bleve.Index) - } - indexNameMapping[name] = idx - indexStats[name] = idx.Stats() -} - -func UnregisterIndexByName(name string) bleve.Index { - indexNameMappingLock.Lock() - defer indexNameMappingLock.Unlock() - - if indexNameMapping == nil { - return nil - } - rv := indexNameMapping[name] - if rv != nil { - delete(indexNameMapping, name) - } - delete(indexStats, name) - return rv -} - -func IndexByName(name string) bleve.Index { - indexNameMappingLock.RLock() - defer indexNameMappingLock.RUnlock() - - return indexNameMapping[name] -} - -func IndexNames() []string { - indexNameMappingLock.RLock() - defer indexNameMappingLock.RUnlock() - - rv := make([]string, len(indexNameMapping)) - count := 0 - for k := range indexNameMapping { - rv[count] = k - count++ - } - return rv -} - -func IndexStats() bleve.IndexStats { - return indexStats -} - -func UpdateAlias(alias string, add, remove []string) error { - indexNameMappingLock.Lock() - defer indexNameMappingLock.Unlock() - - index, exists := indexNameMapping[alias] - if !exists { - // new alias - if len(remove) > 0 { - return fmt.Errorf("cannot remove indexes from a new alias") - } - indexes := make([]bleve.Index, len(add)) - for i, addIndexName := range add { - addIndex, indexExists := indexNameMapping[addIndexName] - if !indexExists { - return fmt.Errorf("index named '%s' does not exist", addIndexName) - } - indexes[i] = addIndex - } - indexAlias := bleve.NewIndexAlias(indexes...) - indexNameMapping[alias] = indexAlias - } else { - // something with this name already exists - indexAlias, isAlias := index.(bleve.IndexAlias) - if !isAlias { - return fmt.Errorf("'%s' is not an alias", alias) - } - // build list of add indexes - addIndexes := make([]bleve.Index, len(add)) - for i, addIndexName := range add { - addIndex, indexExists := indexNameMapping[addIndexName] - if !indexExists { - return fmt.Errorf("index named '%s' does not exist", addIndexName) - } - addIndexes[i] = addIndex - } - // build list of remove indexes - removeIndexes := make([]bleve.Index, len(remove)) - for i, removeIndexName := range remove { - removeIndex, indexExists := indexNameMapping[removeIndexName] - if !indexExists { - return fmt.Errorf("index named '%s' does not exist", removeIndexName) - } - removeIndexes[i] = removeIndex - } - indexAlias.Swap(addIndexes, removeIndexes) - } - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/http/search.go b/vendor/github.com/blevesearch/bleve/http/search.go deleted file mode 100644 index a1448f3a..00000000 --- a/vendor/github.com/blevesearch/bleve/http/search.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - - "github.com/blevesearch/bleve" -) - -// SearchHandler can handle search requests sent over HTTP -type SearchHandler struct { - defaultIndexName string - IndexNameLookup varLookupFunc -} - -func NewSearchHandler(defaultIndexName string) *SearchHandler { - return &SearchHandler{ - defaultIndexName: defaultIndexName, - } -} - -func (h *SearchHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - - // find the index to operate on - var indexName string - if h.IndexNameLookup != nil { - indexName = h.IndexNameLookup(req) - } - if indexName == "" { - indexName = h.defaultIndexName - } - index := IndexByName(indexName) - if index == nil { - showError(w, req, fmt.Sprintf("no such index '%s'", indexName), 404) - return - } - - // read the request body - requestBody, err := ioutil.ReadAll(req.Body) - if err != nil { - showError(w, req, fmt.Sprintf("error reading request body: %v", err), 400) - return - } - - logger.Printf("request body: %s", requestBody) - - // parse the request - var searchRequest bleve.SearchRequest - err = json.Unmarshal(requestBody, &searchRequest) - if err != nil { - showError(w, req, fmt.Sprintf("error parsing query: %v", err), 400) - return - } - - logger.Printf("parsed request %#v", searchRequest) - - // validate the query - err = searchRequest.Query.Validate() - if err != nil { - showError(w, req, fmt.Sprintf("error validating query: %v", err), 400) - return - } - - // execute the query - searchResponse, err := index.Search(&searchRequest) - if err != nil { - showError(w, req, fmt.Sprintf("error executing query: %v", err), 500) - return - } - - // encode the response - mustEncode(w, searchResponse) -} diff --git a/vendor/github.com/blevesearch/bleve/http/util.go b/vendor/github.com/blevesearch/bleve/http/util.go deleted file mode 100644 index 845df7cc..00000000 --- a/vendor/github.com/blevesearch/bleve/http/util.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package http - -import ( - "encoding/json" - "io" - "io/ioutil" - "log" - "net/http" -) - -func showError(w http.ResponseWriter, r *http.Request, - msg string, code int) { - logger.Printf("Reporting error %v/%v", code, msg) - http.Error(w, msg, code) -} - -func mustEncode(w io.Writer, i interface{}) { - if headered, ok := w.(http.ResponseWriter); ok { - headered.Header().Set("Cache-Control", "no-cache") - headered.Header().Set("Content-type", "application/json") - } - - e := json.NewEncoder(w) - if err := e.Encode(i); err != nil { - panic(err) - } -} - -type varLookupFunc func(req *http.Request) string - -var logger = log.New(ioutil.Discard, "bleve.http", log.LstdFlags) - -// SetLog sets the logger used for logging -// by default log messages are sent to ioutil.Discard -func SetLog(l *log.Logger) { - logger = l -} diff --git a/vendor/github.com/blevesearch/bleve/index.go b/vendor/github.com/blevesearch/bleve/index.go index 9514c972..0fadc573 100644 --- a/vendor/github.com/blevesearch/bleve/index.go +++ b/vendor/github.com/blevesearch/bleve/index.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve @@ -13,11 +18,16 @@ import ( "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/mapping" + "golang.org/x/net/context" ) // A Batch groups together multiple Index and Delete // operations you would like performed at the same -// time. +// time. The Batch structure is NOT thread-safe. +// You should only perform operations on a batch +// from a single thread at a time. Once batch +// execution has started, you may not modify it. type Batch struct { index Index internal *index.Batch @@ -27,8 +37,11 @@ type Batch struct { // batch. NOTE: the bleve Index is not updated // until the batch is executed. func (b *Batch) Index(id string, data interface{}) error { + if id == "" { + return ErrorEmptyID + } doc := document.NewDocument(id) - err := b.index.Mapping().mapDocument(doc, data) + err := b.index.Mapping().MapDocument(doc, data) if err != nil { return err } @@ -40,7 +53,9 @@ func (b *Batch) Index(id string, data interface{}) error { // batch. NOTE: the bleve Index is not updated until // the batch is executed. func (b *Batch) Delete(id string) { - b.internal.Delete(id) + if id != "" { + b.internal.Delete(id) + } } // SetInternal adds the specified set internal @@ -63,26 +78,98 @@ func (b *Batch) Size() int { return len(b.internal.IndexOps) + len(b.internal.InternalOps) } -// String prints a user friendly string represenation of what +// String prints a user friendly string representation of what // is inside this batch. func (b *Batch) String() string { return b.internal.String() } +// Reset returns a Batch to the empty state so that it can +// be re-used in the future. +func (b *Batch) Reset() { + b.internal.Reset() +} + // An Index implements all the indexing and searching // capabilities of bleve. An Index can be created // using the New() and Open() methods. +// +// Index() takes an input value, deduces a DocumentMapping for its type, +// assigns string paths to its fields or values then applies field mappings on +// them. +// +// The DocumentMapping used to index a value is deduced by the following rules: +// 1) If value implements Classifier interface, resolve the mapping from Type(). +// 2) If value has a string field or value at IndexMapping.TypeField. +// (defaulting to "_type"), use it to resolve the mapping. Fields addressing +// is described below. +// 3) If IndexMapping.DefaultType is registered, return it. +// 4) Return IndexMapping.DefaultMapping. +// +// Each field or nested field of the value is identified by a string path, then +// mapped to one or several FieldMappings which extract the result for analysis. +// +// Struct values fields are identified by their "json:" tag, or by their name. +// Nested fields are identified by prefixing with their parent identifier, +// separated by a dot. +// +// Map values entries are identified by their string key. Entries not indexed +// by strings are ignored. Entry values are identified recursively like struct +// fields. +// +// Slice and array values are identified by their field name. Their elements +// are processed sequentially with the same FieldMapping. +// +// String, float64 and time.Time values are identified by their field name. +// Other types are ignored. +// +// Each value identifier is decomposed in its parts and recursively address +// SubDocumentMappings in the tree starting at the root DocumentMapping. If a +// mapping is found, all its FieldMappings are applied to the value. If no +// mapping is found and the root DocumentMapping is dynamic, default mappings +// are used based on value type and IndexMapping default configurations. +// +// Finally, mapped values are analyzed, indexed or stored. See +// FieldMapping.Analyzer to know how an analyzer is resolved for a given field. +// +// Examples: +// +// type Date struct { +// Day string `json:"day"` +// Month string +// Year string +// } +// +// type Person struct { +// FirstName string `json:"first_name"` +// LastName string +// BirthDate Date `json:"birth_date"` +// } +// +// A Person value FirstName is mapped by the SubDocumentMapping at +// "first_name". Its LastName is mapped by the one at "LastName". The day of +// BirthDate is mapped to the SubDocumentMapping "day" of the root +// SubDocumentMapping "birth_date". It will appear as the "birth_date.day" +// field in the index. The month is mapped to "birth_date.Month". type Index interface { + // Index analyzes, indexes or stores mapped data fields. Supplied + // identifier is bound to analyzed data and will be retrieved by search + // requests. See Index interface documentation for details about mapping + // rules. Index(id string, data interface{}) error Delete(id string) error NewBatch() *Batch Batch(b *Batch) error + // Document returns specified document or nil if the document is not + // indexed or stored. Document(id string) (*document.Document, error) + // DocCount returns the number of documents in the index. DocCount() (uint64, error) Search(req *SearchRequest) (*SearchResult, error) + SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) Fields() ([]string, error) @@ -90,45 +177,55 @@ type Index interface { FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) - DumpAll() chan interface{} - DumpDoc(id string) chan interface{} - DumpFields() chan interface{} - Close() error - Mapping() *IndexMapping + Mapping() mapping.IndexMapping Stats() *IndexStat + StatsMap() map[string]interface{} GetInternal(key []byte) ([]byte, error) SetInternal(key, val []byte) error DeleteInternal(key []byte) error - Advanced() (index.Index, store.KVStore, error) -} + // Name returns the name of the index (by default this is the path) + Name() string + // SetName lets you assign your own logical name to this index + SetName(string) -// A Classifier is an interface describing any object -// which knows how to identify its own type. -type Classifier interface { - Type() string + // Advanced returns the indexer and data store, exposing lower level + // methods to enumerate records and access data. + Advanced() (index.Index, store.KVStore, error) } // New index at the specified path, must not exist. // The provided mapping will be used for all // Index/Search operations. -func New(path string, mapping *IndexMapping) (Index, error) { - return newIndexUsing(path, mapping, Config.DefaultKVStore, nil) +func New(path string, mapping mapping.IndexMapping) (Index, error) { + return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil) +} + +// NewMemOnly creates a memory-only index. +// The contents of the index is NOT persisted, +// and will be lost once closed. +// The provided mapping will be used for all +// Index/Search operations. +func NewMemOnly(mapping mapping.IndexMapping) (Index, error) { + return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) } // NewUsing creates index at the specified path, // which must not already exist. // The provided mapping will be used for all // Index/Search operations. -// The specified kvstore implemenation will be used +// The specified index type will be used. +// The specified kvstore implementation will be used // and the provided kvconfig will be passed to its -// constructor. -func NewUsing(path string, mapping *IndexMapping, kvstore string, kvconfig map[string]interface{}) (Index, error) { - return newIndexUsing(path, mapping, kvstore, kvconfig) +// constructor. Note that currently the values of kvconfig must +// be able to be marshaled and unmarshaled using the encoding/json library (used +// when reading/writing the index metadata file). +func NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) { + return newIndexUsing(path, mapping, indexType, kvstore, kvconfig) } // Open index at the specified path, must exist. diff --git a/vendor/github.com/blevesearch/bleve/index/analysis.go b/vendor/github.com/blevesearch/bleve/index/analysis.go new file mode 100644 index 00000000..b626b9f3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/analysis.go @@ -0,0 +1,83 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import "github.com/blevesearch/bleve/document" + +type IndexRow interface { + KeySize() int + KeyTo([]byte) (int, error) + Key() []byte + + ValueSize() int + ValueTo([]byte) (int, error) + Value() []byte +} + +type AnalysisResult struct { + DocID string + Rows []IndexRow +} + +type AnalysisWork struct { + i Index + d *document.Document + rc chan *AnalysisResult +} + +func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork { + return &AnalysisWork{ + i: i, + d: d, + rc: rc, + } +} + +type AnalysisQueue struct { + queue chan *AnalysisWork + done chan struct{} +} + +func (q *AnalysisQueue) Queue(work *AnalysisWork) { + q.queue <- work +} + +func (q *AnalysisQueue) Close() { + close(q.done) +} + +func NewAnalysisQueue(numWorkers int) *AnalysisQueue { + rv := AnalysisQueue{ + queue: make(chan *AnalysisWork), + done: make(chan struct{}), + } + for i := 0; i < numWorkers; i++ { + go AnalysisWorker(rv) + } + return &rv +} + +func AnalysisWorker(q AnalysisQueue) { + // read work off the queue + for { + select { + case <-q.done: + return + case w := <-q.queue: + r := w.i.Analyze(w.d) + w.rc <- r + } + } +} diff --git a/vendor/github.com/blevesearch/bleve/index/field_cache.go b/vendor/github.com/blevesearch/bleve/index/field_cache.go new file mode 100644 index 00000000..9354081f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/field_cache.go @@ -0,0 +1,88 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package index + +import ( + "sync" +) + +type FieldCache struct { + fieldIndexes map[string]uint16 + indexFields []string + lastFieldIndex int + mutex sync.RWMutex +} + +func NewFieldCache() *FieldCache { + return &FieldCache{ + fieldIndexes: make(map[string]uint16), + lastFieldIndex: -1, + } +} + +func (f *FieldCache) AddExisting(field string, index uint16) { + f.mutex.Lock() + f.addLOCKED(field, index) + f.mutex.Unlock() +} + +func (f *FieldCache) addLOCKED(field string, index uint16) uint16 { + f.fieldIndexes[field] = index + if len(f.indexFields) < int(index)+1 { + prevIndexFields := f.indexFields + f.indexFields = make([]string, int(index)+16) + copy(f.indexFields, prevIndexFields) + } + f.indexFields[int(index)] = field + if int(index) > f.lastFieldIndex { + f.lastFieldIndex = int(index) + } + return index +} + +// FieldNamed returns the index of the field, and whether or not it existed +// before this call. if createIfMissing is true, and new field index is assigned +// but the second return value will still be false +func (f *FieldCache) FieldNamed(field string, createIfMissing bool) (uint16, bool) { + f.mutex.RLock() + if index, ok := f.fieldIndexes[field]; ok { + f.mutex.RUnlock() + return index, true + } else if !createIfMissing { + f.mutex.RUnlock() + return 0, false + } + // trade read lock for write lock + f.mutex.RUnlock() + f.mutex.Lock() + // need to check again with write lock + if index, ok := f.fieldIndexes[field]; ok { + f.mutex.Unlock() + return index, true + } + // assign next field id + index := f.addLOCKED(field, uint16(f.lastFieldIndex+1)) + f.mutex.Unlock() + return index, false +} + +func (f *FieldCache) FieldIndexed(index uint16) (field string) { + f.mutex.RLock() + if int(index) < len(f.indexFields) { + field = f.indexFields[int(index)] + } + f.mutex.RUnlock() + return field +} diff --git a/vendor/github.com/blevesearch/bleve/index/index.go b/vendor/github.com/blevesearch/bleve/index/index.go index 72c3eaa4..9870b417 100644 --- a/vendor/github.com/blevesearch/bleve/index/index.go +++ b/vendor/github.com/blevesearch/bleve/index/index.go @@ -1,27 +1,34 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package index import ( + "bytes" "encoding/json" "fmt" "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index/store" ) +var ErrorUnknownStorageType = fmt.Errorf("unknown storage type") + type Index interface { Open() error Close() error - DocCount() (uint64, error) - Update(doc *document.Document) error Delete(id string) error Batch(batch *Batch) error @@ -29,55 +36,129 @@ type Index interface { SetInternal(key, val []byte) error DeleteInternal(key []byte) error - DumpAll() chan interface{} - DumpDoc(id string) chan interface{} - DumpFields() chan interface{} - + // Reader returns a low-level accessor on the index data. Close it to + // release associated resources. Reader() (IndexReader, error) Stats() json.Marshaler + StatsMap() map[string]interface{} + + Analyze(d *document.Document) *AnalysisResult + + Advanced() (store.KVStore, error) } +type DocumentFieldTermVisitor func(field string, term []byte) + type IndexReader interface { - TermFieldReader(term []byte, field string) (TermFieldReader, error) - DocIDReader(start, end string) (DocIDReader, error) + TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error) + + // DocIDReader returns an iterator over all doc ids + // The caller must close returned instance to release associated resources. + DocIDReaderAll() (DocIDReader, error) + + DocIDReaderOnly(ids []string) (DocIDReader, error) FieldDict(field string) (FieldDict, error) + + // FieldDictRange is currently defined to include the start and end terms FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error) FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error) Document(id string) (*document.Document, error) - DocumentFieldTerms(id string) (FieldTerms, error) + DocumentVisitFieldTerms(id IndexInternalID, fields []string, visitor DocumentFieldTermVisitor) error Fields() ([]string, error) GetInternal(key []byte) ([]byte, error) - DocCount() uint64 + DocCount() (uint64, error) + + ExternalID(id IndexInternalID) (string, error) + InternalID(id string) (IndexInternalID, error) + + DumpAll() chan interface{} + DumpDoc(id string) chan interface{} + DumpFields() chan interface{} Close() error } +// FieldTerms contains the terms used by a document, keyed by field type FieldTerms map[string][]string +// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields +func (f FieldTerms) FieldsNotYetCached(fields []string) []string { + rv := make([]string, 0, len(fields)) + for _, field := range fields { + if _, ok := f[field]; !ok { + rv = append(rv, field) + } + } + return rv +} + +// Merge will combine two FieldTerms +// it assumes that the terms lists are complete (thus do not need to be merged) +// field terms from the other list always replace the ones in the receiver +func (f FieldTerms) Merge(other FieldTerms) { + for field, terms := range other { + f[field] = terms + } +} + type TermFieldVector struct { - Field string - Pos uint64 - Start uint64 - End uint64 + Field string + ArrayPositions []uint64 + Pos uint64 + Start uint64 + End uint64 +} + +// IndexInternalID is an opaque document identifier interal to the index impl +type IndexInternalID []byte + +func (id IndexInternalID) Equals(other IndexInternalID) bool { + return id.Compare(other) == 0 +} + +func (id IndexInternalID) Compare(other IndexInternalID) int { + return bytes.Compare(id, other) } type TermFieldDoc struct { Term string - ID string + ID IndexInternalID Freq uint64 Norm float64 Vectors []*TermFieldVector } +// Reset allows an already allocated TermFieldDoc to be reused +func (tfd *TermFieldDoc) Reset() *TermFieldDoc { + // remember the []byte used for the ID + id := tfd.ID + // idiom to copy over from empty TermFieldDoc (0 allocations) + *tfd = TermFieldDoc{} + // reuse the []byte already allocated (and reset len to 0) + tfd.ID = id[:0] + return tfd +} + +// TermFieldReader is the interface exposing the enumeration of documents +// containing a given term in a given field. Documents are returned in byte +// lexicographic order over their identifiers. type TermFieldReader interface { - Next() (*TermFieldDoc, error) - Advance(ID string) (*TermFieldDoc, error) + // Next returns the next document containing the term in this field, or nil + // when it reaches the end of the enumeration. The preAlloced TermFieldDoc + // is optional, and when non-nil, will be used instead of allocating memory. + Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Advance resets the enumeration at specified document or its immediate + // follower. + Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Count returns the number of documents contains the term in this field. Count() uint64 Close() error } @@ -92,9 +173,18 @@ type FieldDict interface { Close() error } +// DocIDReader is the interface exposing enumeration of documents identifiers. +// Close the reader to release associated resources. type DocIDReader interface { - Next() (string, error) - Advance(ID string) (string, error) + // Next returns the next document internal identifier in the natural + // index order, nil when the end of the sequence is reached. + Next() (IndexInternalID, error) + + // Advance resets the iteration to the first internal identifier greater than + // or equal to ID. If ID is smaller than the start of the range, the iteration + // will start there instead. If ID is greater than or equal to the end of + // the range, Next() call will return io.EOF. + Advance(ID IndexInternalID) (IndexInternalID, error) Close() error } @@ -144,3 +234,8 @@ func (b *Batch) String() string { } return rv } + +func (b *Batch) Reset() { + b.IndexOps = make(map[string]*document.Document) + b.InternalOps = make(map[string][]byte) +} diff --git a/vendor/github.com/blevesearch/bleve/index/store/batch.go b/vendor/github.com/blevesearch/bleve/index/store/batch.go index 4a3c76bc..71105266 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/batch.go +++ b/vendor/github.com/blevesearch/bleve/index/store/batch.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package store @@ -15,53 +20,41 @@ type op struct { } type EmulatedBatch struct { - w KVWriter - ops []*op - merge *EmulatedMerge + Ops []*op + Merger *EmulatedMerge } -func NewEmulatedBatch(w KVWriter, mo MergeOperator) *EmulatedBatch { +func NewEmulatedBatch(mo MergeOperator) *EmulatedBatch { return &EmulatedBatch{ - w: w, - ops: make([]*op, 0, 1000), - merge: NewEmulatedMerge(mo), + Ops: make([]*op, 0, 1000), + Merger: NewEmulatedMerge(mo), } } func (b *EmulatedBatch) Set(key, val []byte) { - b.ops = append(b.ops, &op{key, val}) + ck := make([]byte, len(key)) + copy(ck, key) + cv := make([]byte, len(val)) + copy(cv, val) + b.Ops = append(b.Ops, &op{ck, cv}) } func (b *EmulatedBatch) Delete(key []byte) { - b.ops = append(b.ops, &op{key, nil}) + ck := make([]byte, len(key)) + copy(ck, key) + b.Ops = append(b.Ops, &op{ck, nil}) } func (b *EmulatedBatch) Merge(key, val []byte) { - b.merge.Merge(key, val) + ck := make([]byte, len(key)) + copy(ck, key) + cv := make([]byte, len(val)) + copy(cv, val) + b.Merger.Merge(key, val) } -func (b *EmulatedBatch) Execute() error { - // first process merges - err := b.merge.Execute(b.w) - if err != nil { - return err - } - - // now apply all the ops - for _, op := range b.ops { - if op.V != nil { - err := b.w.Set(op.K, op.V) - if err != nil { - return err - } - } else { - err := b.w.Delete(op.K) - if err != nil { - return err - } - } - } - return nil +func (b *EmulatedBatch) Reset() { + b.Ops = b.Ops[:0] } func (b *EmulatedBatch) Close() error { diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go index 0cf87049..82ab946f 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go +++ b/vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go @@ -1,15 +1,22 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package boltdb import ( + "bytes" + "github.com/boltdb/bolt" ) @@ -17,24 +24,44 @@ type Iterator struct { store *Store tx *bolt.Tx cursor *bolt.Cursor + prefix []byte + start []byte + end []byte valid bool key []byte val []byte } -func (i *Iterator) SeekFirst() { - i.key, i.val = i.cursor.First() +func (i *Iterator) updateValid() { i.valid = (i.key != nil) + if i.valid { + if i.prefix != nil { + i.valid = bytes.HasPrefix(i.key, i.prefix) + } else if i.end != nil { + i.valid = bytes.Compare(i.key, i.end) < 0 + } + } } func (i *Iterator) Seek(k []byte) { + if i.start != nil && bytes.Compare(k, i.start) < 0 { + k = i.start + } + if i.prefix != nil && !bytes.HasPrefix(k, i.prefix) { + if bytes.Compare(k, i.prefix) < 0 { + k = i.prefix + } else { + i.valid = false + return + } + } i.key, i.val = i.cursor.Seek(k) - i.valid = (i.key != nil) + i.updateValid() } func (i *Iterator) Next() { i.key, i.val = i.cursor.Next() - i.valid = (i.key != nil) + i.updateValid() } func (i *Iterator) Current() ([]byte, []byte, bool) { diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go index 2179ebb5..1d701c98 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go +++ b/vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package boltdb @@ -15,30 +20,51 @@ import ( ) type Reader struct { - store *Store - tx *bolt.Tx -} - -func (r *Reader) BytesSafeAfterClose() bool { - return false + store *Store + tx *bolt.Tx + bucket *bolt.Bucket } func (r *Reader) Get(key []byte) ([]byte, error) { - rv := r.tx.Bucket([]byte(r.store.bucket)).Get(key) + var rv []byte + v := r.bucket.Get(key) + if v != nil { + rv = make([]byte, len(v)) + copy(rv, v) + } return rv, nil } -func (r *Reader) Iterator(key []byte) store.KVIterator { - b := r.tx.Bucket([]byte(r.store.bucket)) - cursor := b.Cursor() +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator { + cursor := r.bucket.Cursor() rv := &Iterator{ store: r.store, tx: r.tx, cursor: cursor, + prefix: prefix, } - rv.Seek(key) + rv.Seek(prefix) + return rv +} + +func (r *Reader) RangeIterator(start, end []byte) store.KVIterator { + cursor := r.bucket.Cursor() + + rv := &Iterator{ + store: r.store, + tx: r.tx, + cursor: cursor, + start: start, + end: end, + } + + rv.Seek(start) return rv } diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go new file mode 100644 index 00000000..e50e5527 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go @@ -0,0 +1,26 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package boltdb + +import "encoding/json" + +type stats struct { + s *Store +} + +func (s *stats) MarshalJSON() ([]byte, error) { + bs := s.s.db.Stats() + return json.Marshal(bs) +} diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go index cd03cd16..d8de0768 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go +++ b/vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go @@ -1,63 +1,105 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Package boltdb implements a store.KVStore on top of BoltDB. It supports the +// following options: +// +// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve". +// +// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index +// operations in exchange of losing integrity guarantees if indexation aborts +// without closing the index. Use it when rebuilding indexes from zero. package boltdb import ( + "bytes" + "encoding/json" "fmt" - "sync" + "os" "github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/registry" "github.com/boltdb/bolt" ) -const Name = "boltdb" +const ( + Name = "boltdb" + defaultCompactBatchSize = 100 +) type Store struct { - path string - bucket string - db *bolt.DB - writer sync.Mutex - mo store.MergeOperator + path string + bucket string + db *bolt.DB + noSync bool + fillPercent float64 + mo store.MergeOperator } -func New(path string, bucket string) *Store { +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path == "" { + return nil, os.ErrInvalid + } + + bucket, ok := config["bucket"].(string) + if !ok { + bucket = "bleve" + } + + noSync, _ := config["nosync"].(bool) + + fillPercent, ok := config["fillPercent"].(float64) + if !ok { + fillPercent = bolt.DefaultFillPercent + } + + bo := &bolt.Options{} + ro, ok := config["read_only"].(bool) + if ok { + bo.ReadOnly = ro + } + + db, err := bolt.Open(path, 0600, bo) + if err != nil { + return nil, err + } + db.NoSync = noSync + + if !bo.ReadOnly { + err = db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bucket)) + + return err + }) + if err != nil { + return nil, err + } + } + rv := Store{ - path: path, - bucket: bucket, + path: path, + bucket: bucket, + db: db, + mo: mo, + noSync: noSync, + fillPercent: fillPercent, } - return &rv -} - -func (bs *Store) Open() error { - - var err error - bs.db, err = bolt.Open(bs.path, 0600, nil) - if err != nil { - return err - } - - err = bs.db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte(bs.bucket)) - - return err - }) - if err != nil { - return err - } - - return nil -} - -func (bs *Store) SetMergeOperator(mo store.MergeOperator) { - bs.mo = mo + return &rv, nil } func (bs *Store) Close() error { @@ -70,43 +112,64 @@ func (bs *Store) Reader() (store.KVReader, error) { return nil, err } return &Reader{ - store: bs, - tx: tx, + store: bs, + tx: tx, + bucket: tx.Bucket([]byte(bs.bucket)), }, nil } func (bs *Store) Writer() (store.KVWriter, error) { - bs.writer.Lock() - tx, err := bs.db.Begin(true) - if err != nil { - bs.writer.Unlock() - return nil, err - } - reader := &Reader{ - store: bs, - tx: tx, - } return &Writer{ - store: bs, - tx: tx, - reader: reader, + store: bs, }, nil } -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") +func (bs *Store) Stats() json.Marshaler { + return &stats{ + s: bs, } +} - bucket, ok := config["bucket"].(string) - if !ok { - bucket = "bleve" +// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches) +// Removing entries is a workaround for github issue #374. +func (bs *Store) CompactWithBatchSize(batchSize int) error { + for { + cnt := 0 + err := bs.db.Batch(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(bs.bucket)).Cursor() + prefix := []byte("d") + + for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() { + if bytes.Equal(v, []byte{0}) { + cnt++ + if err := c.Delete(); err != nil { + return err + } + if cnt == batchSize { + break + } + } + + } + return nil + }) + if err != nil { + return err + } + + if cnt == 0 { + break + } } + return nil +} - return New(path, bucket), nil +// Compact calls CompactWithBatchSize with a default batch size of 100. This is a workaround +// for github issue #374. +func (bs *Store) Compact() error { + return bs.CompactWithBatchSize(defaultCompactBatchSize) } func init() { - registry.RegisterKVStore(Name, StoreConstructor) + registry.RegisterKVStore(Name, New) } diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store_test.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/store_test.go deleted file mode 100644 index fc07be3a..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/store_test.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package boltdb - -import ( - "os" - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -func TestStore(t *testing.T) { - s := New("test", "bleve") - err := s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - CommonTestKVStore(t, s) -} - -func TestReaderIsolation(t *testing.T) { - s := New("test", "bleve") - err := s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - CommonTestReaderIsolation(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify that we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got nil") - } - - // ensure that the director iterator sees it - count = 0 - it2 := newReader.Iterator([]byte{0}) - defer func() { - err := it2.Close() - if err != nil { - t.Fatal(err) - } - }() - for it2.Valid() { - it2.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it3 := reader.Iterator([]byte{0}) - defer func() { - err := it3.Close() - if err != nil { - t.Fatal(err) - } - }() - for it3.Valid() { - it3.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go b/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go index ec6b951f..f0935745 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go +++ b/vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go @@ -1,50 +1,95 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package boltdb import ( + "fmt" + "github.com/blevesearch/bleve/index/store" - "github.com/boltdb/bolt" ) type Writer struct { - store *Store - tx *bolt.Tx - reader *Reader -} - -func (w *Writer) Set(key, val []byte) error { - return w.tx.Bucket([]byte(w.store.bucket)).Put(key, val) -} - -func (w *Writer) Delete(key []byte) error { - return w.tx.Bucket([]byte(w.store.bucket)).Delete(key) + store *Store } func (w *Writer) NewBatch() store.KVBatch { - return store.NewEmulatedBatch(w, w.store.mo) + return store.NewEmulatedBatch(w.store.mo) +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) { + + emulatedBatch, ok := batch.(*store.EmulatedBatch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + tx, err := w.store.db.Begin(true) + if err != nil { + return + } + // defer function to ensure that once started, + // we either Commit tx or Rollback + defer func() { + // if nothing went wrong, commit + if err == nil { + // careful to catch error here too + err = tx.Commit() + } else { + // caller should see error that caused abort, + // not success or failure of Rollback itself + _ = tx.Rollback() + } + }() + + bucket := tx.Bucket([]byte(w.store.bucket)) + bucket.FillPercent = w.store.fillPercent + + for k, mergeOps := range emulatedBatch.Merger.Merges { + kb := []byte(k) + existingVal := bucket.Get(kb) + mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + err = fmt.Errorf("merge operator returned failure") + return + } + err = bucket.Put(kb, mergedVal) + if err != nil { + return + } + } + + for _, op := range emulatedBatch.Ops { + if op.V != nil { + err = bucket.Put(op.K, op.V) + if err != nil { + return + } + } else { + err = bucket.Delete(op.K) + if err != nil { + return + } + } + } + return } func (w *Writer) Close() error { - w.store.writer.Unlock() - return w.tx.Commit() -} - -func (w *Writer) BytesSafeAfterClose() bool { - return w.reader.BytesSafeAfterClose() -} - -func (w *Writer) Get(key []byte) ([]byte, error) { - return w.reader.Get(key) -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.reader.Iterator(key) + return nil } diff --git a/vendor/github.com/blevesearch/bleve/index/store/cznicb/batch.go b/vendor/github.com/blevesearch/bleve/index/store/cznicb/batch.go deleted file mode 100644 index 02757eca..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/cznicb/batch.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -// +build go1.4 - -package cznicb - -import () - -type op struct { - k []byte - v []byte -} - -type Batch struct { - s *Store - ops []op - merges map[string][][]byte -} - -func (b *Batch) Set(k, v []byte) { - b.ops = append(b.ops, op{k, v}) -} - -func (b *Batch) Delete(k []byte) { - b.ops = append(b.ops, op{k, nil}) -} - -func (b *Batch) Merge(key, val []byte) { - ops, ok := b.merges[string(key)] - if ok && len(ops) > 0 { - last := ops[len(ops)-1] - mergedVal, partialMergeOk := b.s.mo.PartialMerge(key, last, val) - if partialMergeOk { - // replace last entry with the result of the merge - ops[len(ops)-1] = mergedVal - } else { - // could not partial merge, append this to the end - ops = append(ops, val) - } - } else { - ops = [][]byte{val} - } - b.merges[string(key)] = ops -} - -func (b *Batch) Execute() (err error) { - b.s.m.Lock() - defer b.s.m.Unlock() - - t := b.s.t - for key, mergeOps := range b.merges { - k := []byte(key) - t.Put(k, func(oldV interface{}, exists bool) (newV interface{}, write bool) { - ob := []byte(nil) - if exists && oldV != nil { - ob = oldV.([]byte) - } - mergedVal, fullMergeOk := b.s.mo.FullMerge(k, ob, mergeOps) - if !fullMergeOk { - return nil, false - } - return mergedVal, true - }) - } - - for _, op := range b.ops { - if op.v != nil { - t.Set(op.k, op.v) - } else { - t.Delete(op.k) - } - } - - return nil -} - -func (b *Batch) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb.go b/vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb.go deleted file mode 100644 index e1887292..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -// +build go1.4 - -// Package cznicb provides an in-memory implementation of the KVStore -// interfaces using the cznic/b in-memory btree. Of note: this -// implementation does not have reader isolation. -package cznicb - -import ( - "bytes" - "fmt" - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - - "github.com/cznic/b" -) - -const Name = "cznicb" - -const MAX_CONCURRENT_WRITERS = 1 - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - s := &Store{ - t: b.TreeNew(itemCompare), - availableWriters: make(chan bool, MAX_CONCURRENT_WRITERS), - } - for i := 0; i < MAX_CONCURRENT_WRITERS; i++ { - s.availableWriters <- true - } - return s, nil -} - -func itemCompare(a, b interface{}) int { - return bytes.Compare(a.([]byte), b.([]byte)) -} - -type Store struct { - availableWriters chan bool - m sync.RWMutex - t *b.Tree - mo store.MergeOperator -} - -func (s *Store) Open() error { - return nil -} - -func (s *Store) SetMergeOperator(mo store.MergeOperator) { - s.mo = mo -} - -func (s *Store) Reader() (store.KVReader, error) { - return &Reader{s: s}, nil -} - -func (s *Store) Writer() (store.KVWriter, error) { - available, ok := <-s.availableWriters - if !ok || !available { - return nil, fmt.Errorf("no available writers") - } - return &Writer{s: s, r: &Reader{s: s}}, nil -} - -func (s *Store) Close() error { - return nil -} - -func (s *Store) get(k []byte) ([]byte, error) { - s.m.RLock() - defer s.m.RUnlock() - v, ok := s.t.Get(k) - if !ok || v == nil { - return nil, nil - } - return v.([]byte), nil -} - -func (s *Store) iterator(k []byte) store.KVIterator { - iter := &Iterator{s: s} - iter.Seek(k) - return iter -} - -func (s *Store) set(k, v []byte) (err error) { - s.m.Lock() - defer s.m.Unlock() - s.t.Set(k, v) - return nil -} - -func (s *Store) delete(k []byte) (err error) { - s.m.Lock() - defer s.m.Unlock() - s.t.Delete(k) - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb_test.go b/vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb_test.go deleted file mode 100644 index ca48e187..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -package cznicb - -import ( - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -func TestCznicBStore(t *testing.T) { - s, err := StoreConstructor(nil) - if err != nil { - t.Fatal(err) - } - - CommonTestKVStore(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - v, err := writer.Get([]byte("a")) - if err != nil { - t.Fatal(err) - } - if string(v) != "val-a" { - t.Errorf("expected val-a") - } - v, err = writer.Get([]byte("not-there")) - if err != nil { - t.Fatal(err) - } - if v != nil { - t.Errorf("expected nil v") - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/cznicb/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/cznicb/iterator.go deleted file mode 100644 index eadcd7bd..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/cznicb/iterator.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -// +build go1.4 - -package cznicb - -import ( - "errors" - - "github.com/cznic/b" -) - -var iteratorDoneErr = errors.New("iteratorDoneErr") // A sentinel value. - -type Iterator struct { // Assuming that iterators are used single-threaded. - s *Store - e *b.Enumerator - - currK interface{} - currV interface{} - currErr error -} - -func (i *Iterator) SeekFirst() { - i.currK = nil - i.currV = nil - i.currErr = nil - - var err error - i.s.m.RLock() - i.e, err = i.s.t.SeekFirst() - i.s.m.RUnlock() // cannot defer, must unlock before Next - if err != nil { - i.currK = nil - i.currV = nil - i.currErr = iteratorDoneErr - } - - i.Next() -} - -func (i *Iterator) Seek(k []byte) { - i.currK = nil - i.currV = nil - i.currErr = nil - - i.s.m.RLock() - i.e, _ = i.s.t.Seek(k) - i.s.m.RUnlock() // cannot defer, must unlock before Next - - i.Next() -} - -func (i *Iterator) Next() { - if i.currErr != nil { - i.currK = nil - i.currV = nil - i.currErr = iteratorDoneErr - return - } - - i.s.m.RLock() - defer i.s.m.RUnlock() - i.currK, i.currV, i.currErr = i.e.Next() -} - -func (i *Iterator) Current() ([]byte, []byte, bool) { - if i.currErr == iteratorDoneErr || - i.currK == nil || - i.currV == nil { - return nil, nil, false - } - - return i.currK.([]byte), i.currV.([]byte), true -} - -func (i *Iterator) Key() []byte { - k, _, ok := i.Current() - if !ok { - return nil - } - return k -} - -func (i *Iterator) Value() []byte { - _, v, ok := i.Current() - if !ok { - return nil - } - return v -} - -func (i *Iterator) Valid() bool { - _, _, ok := i.Current() - return ok -} - -func (i *Iterator) Close() error { - if i.e != nil { - i.e.Close() - } - i.e = nil - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/cznicb/reader.go b/vendor/github.com/blevesearch/bleve/index/store/cznicb/reader.go deleted file mode 100644 index 4956f7dd..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/cznicb/reader.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -// +build go1.4 - -package cznicb - -import ( - "github.com/blevesearch/bleve/index/store" -) - -type Reader struct { - s *Store -} - -func newReader(s *Store) (*Reader, error) { - return &Reader{ - s: s, - }, nil -} - -func (r *Reader) BytesSafeAfterClose() bool { - return false -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - return r.s.get(key) -} - -func (r *Reader) Iterator(key []byte) store.KVIterator { - return r.s.iterator(key) -} - -func (r *Reader) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/cznicb/writer.go b/vendor/github.com/blevesearch/bleve/index/store/cznicb/writer.go deleted file mode 100644 index 279d17a7..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/cznicb/writer.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build go1.4 - -package cznicb - -import ( - "github.com/blevesearch/bleve/index/store" -) - -type Writer struct { - s *Store - r *Reader -} - -func (w *Writer) BytesSafeAfterClose() bool { - return false -} - -func (w *Writer) Set(key, val []byte) error { - return w.s.set(key, val) -} - -func (w *Writer) Delete(key []byte) error { - return w.s.delete(key) -} - -func (w *Writer) NewBatch() store.KVBatch { - return &Batch{ - s: w.s, - ops: make([]op, 0, 1000), - merges: make(map[string][][]byte), - } -} - -func (w *Writer) Close() error { - w.s.availableWriters <- true - w.s = nil - return nil -} - -func (w *Writer) Get(key []byte) ([]byte, error) { - return w.r.s.get(key) -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.r.s.iterator(key) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/forestdb/batch.go b/vendor/github.com/blevesearch/bleve/index/store/forestdb/batch.go deleted file mode 100644 index a8fad295..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/forestdb/batch.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package forestdb - -import ( - "fmt" -) - -type op struct { - k []byte - v []byte -} - -type Batch struct { - s *Store - ops []op - merges map[string][][]byte -} - -func (b *Batch) Set(k, v []byte) { - b.ops = append(b.ops, op{k, v}) -} - -func (b *Batch) Delete(k []byte) { - b.ops = append(b.ops, op{k, nil}) -} - -func (b *Batch) Merge(key, val []byte) { - ops, ok := b.merges[string(key)] - if ok && len(ops) > 0 { - last := ops[len(ops)-1] - mergedVal, partialMergeOk := b.s.mo.PartialMerge(key, last, val) - if partialMergeOk { - // replace last entry with the result of the merge - ops[len(ops)-1] = mergedVal - } else { - // could not partial merge, append this to the end - ops = append(ops, val) - } - } else { - ops = [][]byte{val} - } - b.merges[string(key)] = ops -} - -func (b *Batch) Execute() (err error) { - - for k, mergeOps := range b.merges { - kb := []byte(k) - existingVal, err := b.s.get(kb) - if err != nil { - return err - } - mergedVal, fullMergeOk := b.s.mo.FullMerge(kb, existingVal, mergeOps) - if !fullMergeOk { - return fmt.Errorf("merge operator returned failure") - } - err = b.s.setlocked(kb, mergedVal) - if err != nil { - return err - } - } - - for _, op := range b.ops { - if op.v != nil { - b.s.setlocked(op.k, op.v) - } else { - b.s.deletelocked(op.k) - } - } - - return b.s.commit() -} - -func (b *Batch) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/forestdb/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/forestdb/iterator.go deleted file mode 100644 index 5210352d..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/forestdb/iterator.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package forestdb - -import ( - "github.com/couchbase/goforestdb" -) - -type Iterator struct { - store *Store - snapshot *forestdb.KVStore - iterator *forestdb.Iterator - curr *forestdb.Doc - valid bool -} - -func newIterator(store *Store) *Iterator { - itr, err := store.dbkv.IteratorInit([]byte{}, nil, forestdb.ITR_NONE) - rv := Iterator{ - store: store, - iterator: itr, - valid: err == nil, - } - return &rv -} - -func newIteratorWithSnapshot(store *Store, snapshot *forestdb.KVStore) *Iterator { - itr, err := snapshot.IteratorInit([]byte{}, nil, forestdb.ITR_NONE) - rv := Iterator{ - store: store, - iterator: itr, - valid: err == nil, - } - return &rv -} - -func (i *Iterator) SeekFirst() { - err := i.iterator.SeekMin() - if err != nil { - i.valid = false - return - } - if i.curr != nil { - i.curr.Close() - } - i.curr, err = i.iterator.Get() - if err != nil { - i.valid = false - } -} - -func (i *Iterator) Seek(key []byte) { - err := i.iterator.Seek(key, forestdb.FDB_ITR_SEEK_HIGHER) - if err != nil { - i.valid = false - return - } - if i.curr != nil { - i.curr.Close() - } - i.curr, err = i.iterator.Get() - if err != nil { - i.valid = false - return - } -} - -func (i *Iterator) Next() { - err := i.iterator.Next() - if err != nil { - i.valid = false - return - } - if i.curr != nil { - i.curr.Close() - } - i.curr, err = i.iterator.Get() - if err != nil { - i.valid = false - } -} - -func (i *Iterator) Current() ([]byte, []byte, bool) { - if i.Valid() { - return i.Key(), i.Value(), true - } - return nil, nil, false -} - -func (i *Iterator) Key() []byte { - return i.curr.Key() -} - -func (i *Iterator) Value() []byte { - return i.curr.Body() -} - -func (i *Iterator) Valid() bool { - return i.valid -} - -func (i *Iterator) Close() error { - i.valid = false - if i.curr != nil { - i.curr.Close() - } - return i.iterator.Close() -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/forestdb/reader.go b/vendor/github.com/blevesearch/bleve/index/store/forestdb/reader.go deleted file mode 100644 index 76f97875..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/forestdb/reader.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package forestdb - -import ( - "fmt" - - "github.com/blevesearch/bleve/index/store" - "github.com/couchbase/goforestdb" -) - -type Reader struct { - store *Store - snapshot *forestdb.KVStore -} - -func (r *Reader) BytesSafeAfterClose() bool { - return true -} - -func newReader(store *Store) (*Reader, error) { - snapshot, err := store.newSnapshot() - if err != nil { - return nil, fmt.Errorf("error opening snapshot: %v", err) - } - return &Reader{ - store: store, - snapshot: snapshot, - }, nil -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - res, err := r.snapshot.GetKV(key) - if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND { - return nil, err - } - return res, nil -} - -func (r *Reader) Iterator(key []byte) store.KVIterator { - rv := newIteratorWithSnapshot(r.store, r.snapshot) - rv.Seek(key) - return rv -} - -func (r *Reader) Close() error { - return r.snapshot.Close() -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/forestdb/store.go b/vendor/github.com/blevesearch/bleve/index/store/forestdb/store.go deleted file mode 100644 index e457ad43..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/forestdb/store.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package forestdb - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - "github.com/couchbase/goforestdb" -) - -type ForestDBConfig struct { - BlockSize uint32 - BufferCacheSize uint64 - ChunkSize uint16 - CleanupCacheOnClose bool - CompactionBufferSizeMax uint32 - CompactionMinimumFilesize uint64 - CompactionMode forestdb.CompactOpt - CompactionThreshold uint8 - CompactorSleepDuration uint64 - CompressDocumentBody bool - DurabilityOpt forestdb.DurabilityOpt - OpenFlags forestdb.OpenFlags - PurgingInterval uint32 - SeqTreeOpt forestdb.SeqTreeOpt - WalFlushBeforeCommit bool - WalThreshold uint64 -} - -const Name = "forestdb" - -type Store struct { - path string - config *forestdb.Config - kvconfig *forestdb.KVStoreConfig - dbfile *forestdb.File - dbkv *forestdb.KVStore - writer sync.Mutex - mo store.MergeOperator -} - -func New(path string, createIfMissing bool, - config map[string]interface{}) (*Store, error) { - if config == nil { - config = map[string]interface{}{} - } - - forestDBDefaultConfig := forestdb.DefaultConfig() - forestDBDefaultConfig.SetCompactionMode(forestdb.COMPACT_AUTO) - forestDBConfig, err := applyConfig(forestDBDefaultConfig, config) - if err != nil { - return nil, err - } - - rv := Store{ - path: path, - config: forestDBConfig, - kvconfig: forestdb.DefaultKVStoreConfig(), - } - - if createIfMissing { - rv.kvconfig.SetCreateIfMissing(true) - } - - return &rv, nil -} - -func (s *Store) Open() error { - var err error - s.dbfile, err = forestdb.Open(s.path, s.config) - if err != nil { - return err - } - - s.dbkv, err = s.dbfile.OpenKVStoreDefault(s.kvconfig) - if err != nil { - return err - } - - return nil -} - -func (s *Store) SetMergeOperator(mo store.MergeOperator) { - s.mo = mo -} - -func (s *Store) get(key []byte) ([]byte, error) { - res, err := s.dbkv.GetKV(key) - if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND { - return nil, err - } - return res, nil -} - -func (s *Store) set(key, val []byte) error { - s.writer.Lock() - defer s.writer.Unlock() - return s.setlocked(key, val) -} - -func (s *Store) setlocked(key, val []byte) error { - return s.dbkv.SetKV(key, val) -} - -func (s *Store) delete(key []byte) error { - s.writer.Lock() - defer s.writer.Unlock() - return s.deletelocked(key) -} - -func (s *Store) deletelocked(key []byte) error { - return s.dbkv.DeleteKV(key) -} - -func (s *Store) commit() error { - return s.dbfile.Commit(forestdb.COMMIT_NORMAL) -} - -func (s *Store) Close() error { - err := s.dbkv.Close() - if err != nil { - return err - } - return s.dbfile.Close() - -} - -func (ldbs *Store) iterator(key []byte) store.KVIterator { - rv := newIterator(ldbs) - rv.Seek(key) - return rv -} - -func (s *Store) Reader() (store.KVReader, error) { - return newReader(s) -} - -func (ldbs *Store) Writer() (store.KVWriter, error) { - return newWriter(ldbs) -} - -func (s *Store) getSeqNum() (forestdb.SeqNum, error) { - dbinfo, err := s.dbkv.Info() - if err != nil { - return 0, err - } - return dbinfo.LastSeqNum(), nil -} - -func (s *Store) newSnapshot() (*forestdb.KVStore, error) { - seqNum, err := s.getSeqNum() - if err != nil { - return nil, fmt.Errorf("error getting snapshot seqnum: %v", err) - } - snapshot, err := s.dbkv.SnapshotOpen(seqNum) - if err == forestdb.RESULT_NO_DB_INSTANCE { - checkAgainSeqNum, err := s.getSeqNum() - if err != nil { - return nil, fmt.Errorf("error getting snapshot seqnum again: %v", err) - } - return nil, fmt.Errorf("cannot open snapshot %v, checked again its %v, error: %v", seqNum, checkAgainSeqNum, err) - } - return snapshot, err -} - -func (s *Store) GetRollbackID() ([]byte, error) { - seqNum, err := s.getSeqNum() - if err != nil { - return nil, err - } - buf := new(bytes.Buffer) - err = binary.Write(buf, binary.LittleEndian, seqNum) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func (s *Store) RollbackTo(rollbackId []byte) error { - s.writer.Lock() - defer s.writer.Unlock() - buf := bytes.NewReader(rollbackId) - var seqNum forestdb.SeqNum - err := binary.Read(buf, binary.LittleEndian, &seqNum) - if err != nil { - return err - } - err = s.dbkv.Rollback(seqNum) - if err != nil { - return err - } - return nil -} - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") - } - createIfMissing := false - cim, ok := config["create_if_missing"].(bool) - if ok { - createIfMissing = cim - } - return New(path, createIfMissing, config) -} - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} - -func applyConfig(c *forestdb.Config, config map[string]interface{}) ( - *forestdb.Config, error) { - v, exists := config["forestDBConfig"] - if !exists || v == nil { - return c, nil - } - m, ok := v.(map[string]interface{}) - if !ok { - return c, nil - } - // These extra steps of json.Marshal()/Unmarshal() help to convert - // to the types that we need for the setter calls. - b, err := json.Marshal(m) - if err != nil { - return nil, err - } - var f ForestDBConfig - err = json.Unmarshal(b, &f) - if err != nil { - return nil, err - } - if _, exists := m["blockSize"]; exists { - c.SetBlockSize(f.BlockSize) - } - if _, exists := m["bufferCacheSize"]; exists { - c.SetBufferCacheSize(f.BufferCacheSize) - } - if _, exists := m["chunkSize"]; exists { - c.SetChunkSize(f.ChunkSize) - } - if _, exists := m["cleanupCacheOnClose"]; exists { - c.SetCleanupCacheOnClose(f.CleanupCacheOnClose) - } - if _, exists := m["compactionBufferSizeMax"]; exists { - c.SetCompactionBufferSizeMax(f.CompactionBufferSizeMax) - } - if _, exists := m["compactionMinimumFilesize"]; exists { - c.SetCompactionMinimumFilesize(f.CompactionMinimumFilesize) - } - if _, exists := m["compactionMode"]; exists { - c.SetCompactionMode(f.CompactionMode) - } - if _, exists := m["compactionThreshold"]; exists { - c.SetCompactionThreshold(f.CompactionThreshold) - } - if _, exists := m["compactorSleepDuration"]; exists { - c.SetCompactorSleepDuration(f.CompactorSleepDuration) - } - if _, exists := m["compressDocumentBody"]; exists { - c.SetCompressDocumentBody(f.CompressDocumentBody) - } - if _, exists := m["durabilityOpt"]; exists { - c.SetDurabilityOpt(f.DurabilityOpt) - } - if _, exists := m["openFlags"]; exists { - c.SetOpenFlags(f.OpenFlags) - } - if _, exists := m["purgingInterval"]; exists { - c.SetPurgingInterval(f.PurgingInterval) - } - if _, exists := m["seqTreeOpt"]; exists { - c.SetSeqTreeOpt(f.SeqTreeOpt) - } - if _, exists := m["walFlushBeforeCommit"]; exists { - c.SetWalFlushBeforeCommit(f.WalFlushBeforeCommit) - } - if _, exists := m["walThreshold"]; exists { - c.SetWalThreshold(f.WalThreshold) - } - return c, nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/forestdb/store_test.go b/vendor/github.com/blevesearch/bleve/index/store/forestdb/store_test.go deleted file mode 100644 index dd90fbd9..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/forestdb/store_test.go +++ /dev/null @@ -1,636 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package forestdb - -import ( - "os" - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -func TestForestDBStore(t *testing.T) { - defer func() { - err := os.RemoveAll("testdir") - if err != nil { - t.Fatal(err) - } - }() - - err := os.MkdirAll("testdir", 0700) - if err != nil { - t.Fatal(err) - } - s, err := New("testdir/test", true, nil) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestKVStore(t, s) -} - -func TestReaderIsolation(t *testing.T) { - defer func() { - err := os.RemoveAll("testdir") - if err != nil { - t.Fatal(err) - } - }() - - err := os.MkdirAll("testdir", 0700) - if err != nil { - t.Fatal(err) - } - s, err := New("testdir/test", true, nil) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestReaderIsolation(t, s) -} - -// TestRollbackSameHandle tries to rollback a handle -// and ensure that subsequent reads from it also -// reflect the rollback -func TestRollbackSameHandle(t *testing.T) { - defer func() { - err := os.RemoveAll("testdir") - if err != nil { - t.Fatal(err) - } - }() - - err := os.MkdirAll("testdir", 0700) - if err != nil { - t.Fatal(err) - } - s, err := New("testdir/test", true, nil) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - writer, err := s.Writer() - if err != nil { - t.Fatal(err) - } - - // create 2 docs, a and b - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Error(err) - } - - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Error(err) - } - - // get the rollback id - rollbackId, err := s.GetRollbackID() - if err != nil { - t.Error(err) - } - - // create a 3rd doc c - err = writer.Set([]byte("c"), []byte("val-c")) - if err != nil { - t.Error(err) - } - - err = writer.Close() - if err != nil { - t.Error(err) - } - - // make sure c is there - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - val, err := reader.Get([]byte("c")) - if err != nil { - t.Error(err) - } - if string(val) != "val-c" { - t.Errorf("expected value 'val-c' got '%s'", val) - } - err = reader.Close() - if err != nil { - t.Fatal(err) - } - - // now rollback - err = s.RollbackTo(rollbackId) - if err != nil { - t.Fatal(err) - } - - // now make sure c is not there - reader, err = s.Reader() - if err != nil { - t.Error(err) - } - val, err = reader.Get([]byte("c")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected missing, got '%s'", val) - } - err = reader.Close() - if err != nil { - t.Fatal(err) - } -} - -// TestRollbackNewHandle tries to rollback the -// database, then opens a new handle, and ensures -// that the rollback is reflected there as well -func TestRollbackNewHandle(t *testing.T) { - defer func() { - err := os.RemoveAll("testdir") - if err != nil { - t.Fatal(err) - } - }() - - err := os.MkdirAll("testdir", 0700) - if err != nil { - t.Fatal(err) - } - s, err := New("testdir/test", true, nil) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - writer, err := s.Writer() - if err != nil { - t.Fatal(err) - } - - // create 2 docs, a and b - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Error(err) - } - - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Error(err) - } - - // get the rollback id - rollbackId, err := s.GetRollbackID() - if err != nil { - t.Error(err) - } - - // create a 3rd doc c - err = writer.Set([]byte("c"), []byte("val-c")) - if err != nil { - t.Error(err) - } - - err = writer.Close() - if err != nil { - t.Error(err) - } - - // make sure c is there - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - val, err := reader.Get([]byte("c")) - if err != nil { - t.Error(err) - } - if string(val) != "val-c" { - t.Errorf("expected value 'val-c' got '%s'", val) - } - err = reader.Close() - if err != nil { - t.Fatal(err) - } - - // now rollback - err = s.RollbackTo(rollbackId) - if err != nil { - t.Fatal(err) - } - - // now lets open another handle - s2, err := New("testdir/test", true, nil) - if err != nil { - t.Fatal(err) - } - err = s2.Open() - if err != nil { - t.Fatal(err) - } - defer s2.Close() - - // now make sure c is not there - reader2, err := s2.Reader() - if err != nil { - t.Error(err) - } - val, err = reader2.Get([]byte("c")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected missing, got '%s'", val) - } - err = reader2.Close() - if err != nil { - t.Fatal(err) - } -} - -// TestRollbackOtherHandle tries to create 2 handles -// at the beginning, then rollback one of them -// and ensure it affects the other -func TestRollbackOtherHandle(t *testing.T) { - defer func() { - err := os.RemoveAll("testdir") - if err != nil { - t.Fatal(err) - } - }() - - err := os.MkdirAll("testdir", 0700) - if err != nil { - t.Fatal(err) - } - s, err := New("testdir/test", true, nil) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - // open another handle at the same time - s2, err := New("testdir/test", true, nil) - if err != nil { - t.Fatal(err) - } - err = s2.Open() - if err != nil { - t.Fatal(err) - } - defer s2.Close() - - writer, err := s.Writer() - if err != nil { - t.Fatal(err) - } - - // create 2 docs, a and b - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Error(err) - } - - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Error(err) - } - - // get the rollback id - rollbackId, err := s.GetRollbackID() - if err != nil { - t.Error(err) - } - - // create a 3rd doc c - err = writer.Set([]byte("c"), []byte("val-c")) - if err != nil { - t.Error(err) - } - - err = writer.Close() - if err != nil { - t.Error(err) - } - - // make sure c is there - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - val, err := reader.Get([]byte("c")) - if err != nil { - t.Error(err) - } - if string(val) != "val-c" { - t.Errorf("expected value 'val-c' got '%s'", val) - } - err = reader.Close() - if err != nil { - t.Fatal(err) - } - - // now rollback - err = s.RollbackTo(rollbackId) - if err != nil { - t.Fatal(err) - } - - // now make sure c is not on the other handle - reader2, err := s2.Reader() - if err != nil { - t.Error(err) - } - val, err = reader2.Get([]byte("c")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected missing, got '%s'", val) - } - err = reader2.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got nil") - } - - // ensure that the director iterator sees it - count = 0 - it2 := newReader.Iterator([]byte{0}) - defer func() { - err := it2.Close() - if err != nil { - t.Fatal(err) - } - }() - for it2.Valid() { - it2.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it3 := reader.Iterator([]byte{0}) - defer func() { - err := it3.Close() - if err != nil { - t.Fatal(err) - } - }() - for it3.Valid() { - it3.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/forestdb/writer.go b/vendor/github.com/blevesearch/bleve/index/store/forestdb/writer.go deleted file mode 100644 index 01909738..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/forestdb/writer.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package forestdb - -import ( - "github.com/blevesearch/bleve/index/store" -) - -type Writer struct { - store *Store -} - -func (w *Writer) BytesSafeAfterClose() bool { - return true -} - -func newWriter(store *Store) (*Writer, error) { - store.writer.Lock() - return &Writer{ - store: store, - }, nil -} - -func (w *Writer) Set(key, val []byte) error { - err := w.store.setlocked(key, val) - if err != nil { - return err - } - return w.store.commit() -} - -func (w *Writer) Delete(key []byte) error { - err := w.store.deletelocked(key) - if err != nil { - return err - } - return w.store.commit() -} - -func (w *Writer) NewBatch() store.KVBatch { - return &Batch{ - s: w.store, - ops: make([]op, 0, 1000), - merges: make(map[string][][]byte), - } -} - -func (w *Writer) Close() error { - w.store.writer.Unlock() - return nil -} - -// these two methods can safely read using the regular -// methods without a read transaction, because we know -// that no one else is writing but us -func (w *Writer) Get(key []byte) ([]byte, error) { - return w.store.get(key) -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.store.iterator(key) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/batch.go b/vendor/github.com/blevesearch/bleve/index/store/goleveldb/batch.go deleted file mode 100644 index b56a3bb7..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/batch.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package goleveldb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/syndtr/goleveldb/leveldb" -) - -type Batch struct { - w *Writer - merge *store.EmulatedMerge - batch *leveldb.Batch -} - -func (b *Batch) Set(key, val []byte) { - b.batch.Put(key, val) -} - -func (b *Batch) Delete(key []byte) { - b.batch.Delete(key) -} - -func (b *Batch) Merge(key, val []byte) { - b.merge.Merge(key, val) -} - -func (b *Batch) Execute() error { - - // first process merges - ops, err := b.merge.ExecuteDeferred(b.w) - if err != nil { - return err - } - for _, op := range ops { - b.batch.Put(op.K, op.V) - } - - wopts := defaultWriteOptions() - err = b.w.store.db.Write(b.batch, wopts) - return err -} - -func (b *Batch) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/goleveldb/iterator.go deleted file mode 100644 index f067aeb5..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/iterator.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package goleveldb - -import ( - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/iterator" -) - -type Iterator struct { - store *Store - iterator iterator.Iterator -} - -func newIterator(store *Store) *Iterator { - ropts := defaultReadOptions() - iter := store.db.NewIterator(nil, ropts) - rv := Iterator{ - store: store, - iterator: iter, - } - return &rv -} - -func newIteratorWithSnapshot(store *Store, snapshot *leveldb.Snapshot) *Iterator { - options := defaultReadOptions() - iter := snapshot.NewIterator(nil, options) - rv := Iterator{ - store: store, - iterator: iter, - } - return &rv -} - -func (ldi *Iterator) SeekFirst() { - ldi.iterator.First() -} - -func (ldi *Iterator) Seek(key []byte) { - ldi.iterator.Seek(key) -} - -func (ldi *Iterator) Next() { - ldi.iterator.Next() -} - -func (ldi *Iterator) Current() ([]byte, []byte, bool) { - if ldi.Valid() { - return ldi.Key(), ldi.Value(), true - } - return nil, nil, false -} - -func (ldi *Iterator) Key() []byte { - return ldi.iterator.Key() -} - -func (ldi *Iterator) Value() []byte { - return ldi.iterator.Value() -} - -func (ldi *Iterator) Valid() bool { - return ldi.iterator.Valid() -} - -func (ldi *Iterator) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/reader.go b/vendor/github.com/blevesearch/bleve/index/store/goleveldb/reader.go deleted file mode 100644 index 23316a22..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/reader.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package goleveldb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/syndtr/goleveldb/leveldb" -) - -type Reader struct { - store *Store - snapshot *leveldb.Snapshot -} - -func newReader(store *Store) (*Reader, error) { - snapshot, _ := store.db.GetSnapshot() - return &Reader{ - store: store, - snapshot: snapshot, - }, nil -} - -func (r *Reader) BytesSafeAfterClose() bool { - return false -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - return r.store.getWithSnapshot(key, r.snapshot) -} - -func (r *Reader) Iterator(key []byte) store.KVIterator { - rv := newIteratorWithSnapshot(r.store, r.snapshot) - rv.Seek(key) - return rv -} - -func (r *Reader) Close() error { - r.snapshot.Release() - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/store.go b/vendor/github.com/blevesearch/bleve/index/store/goleveldb/store.go deleted file mode 100644 index 33fbbc4f..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/store.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package goleveldb - -import ( - "fmt" - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/opt" -) - -const Name = "goleveldb" - -type Store struct { - path string - opts *opt.Options - db *leveldb.DB - writer sync.Mutex - mo store.MergeOperator -} - -func New(path string, config map[string]interface{}) (*Store, error) { - rv := Store{ - path: path, - opts: &opt.Options{}, - } - - _, err := applyConfig(rv.opts, config) - if err != nil { - return nil, err - } - - return &rv, nil -} - -func (ldbs *Store) Open() error { - var err error - ldbs.db, err = leveldb.OpenFile(ldbs.path, ldbs.opts) - if err != nil { - return err - } - return nil -} - -func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) { - ldbs.mo = mo -} - -func (ldbs *Store) get(key []byte) ([]byte, error) { - options := defaultReadOptions() - b, err := ldbs.db.Get(key, options) - if err == leveldb.ErrNotFound { - return nil, nil - } - return b, err -} - -func (ldbs *Store) getWithSnapshot(key []byte, snapshot *leveldb.Snapshot) ([]byte, error) { - options := defaultReadOptions() - b, err := snapshot.Get(key, options) - if err == leveldb.ErrNotFound { - return nil, nil - } - return b, err -} - -func (ldbs *Store) set(key, val []byte) error { - ldbs.writer.Lock() - defer ldbs.writer.Unlock() - return ldbs.setlocked(key, val) -} - -func (ldbs *Store) setlocked(key, val []byte) error { - options := defaultWriteOptions() - err := ldbs.db.Put(key, val, options) - return err -} - -func (ldbs *Store) delete(key []byte) error { - ldbs.writer.Lock() - defer ldbs.writer.Unlock() - return ldbs.deletelocked(key) -} - -func (ldbs *Store) deletelocked(key []byte) error { - options := defaultWriteOptions() - err := ldbs.db.Delete(key, options) - return err -} - -func (ldbs *Store) Close() error { - return ldbs.db.Close() -} - -func (ldbs *Store) iterator(key []byte) store.KVIterator { - rv := newIterator(ldbs) - rv.Seek(key) - return rv -} - -func (ldbs *Store) Reader() (store.KVReader, error) { - return newReader(ldbs) -} - -func (ldbs *Store) Writer() (store.KVWriter, error) { - return newWriter(ldbs) -} - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") - } - return New(path, config) -} - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} - -func applyConfig(o *opt.Options, config map[string]interface{}) ( - *opt.Options, error) { - - cim, ok := config["create_if_missing"].(bool) - if ok { - o.ErrorIfMissing = !cim - } - - eie, ok := config["error_if_exists"].(bool) - if ok { - o.ErrorIfExist = eie - } - - wbs, ok := config["write_buffer_size"].(float64) - if ok { - o.WriteBuffer = int(wbs) - } - - bs, ok := config["block_size"].(float64) - if ok { - o.BlockSize = int(bs) - } - - bri, ok := config["block_restart_interval"].(float64) - if ok { - o.BlockRestartInterval = int(bri) - } - - lcc, ok := config["lru_cache_capacity"].(float64) - if ok { - o.BlockCacheCapacity = int(lcc) - } - - bfbpk, ok := config["bloom_filter_bits_per_key"].(float64) - if ok { - bf := filter.NewBloomFilter(int(bfbpk)) - o.Filter = bf - } - - return o, nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/store_test.go b/vendor/github.com/blevesearch/bleve/index/store/goleveldb/store_test.go deleted file mode 100644 index 2020c88e..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/store_test.go +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package goleveldb - -import ( - "os" - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -var leveldbTestOptions = map[string]interface{}{ - "create_if_missing": true, -} - -func TestLevelDBStore(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s, err := New("test", leveldbTestOptions) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestKVStore(t, s) -} - -func TestReaderIsolation(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s, err := New("test", leveldbTestOptions) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestReaderIsolation(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify that we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got nil") - } - - // ensure that the director iterator sees it - count = 0 - it = newReader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it = reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/util.go b/vendor/github.com/blevesearch/bleve/index/store/goleveldb/util.go deleted file mode 100644 index 4e22808b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/util.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package goleveldb - -import ( - "github.com/syndtr/goleveldb/leveldb/opt" -) - -func defaultWriteOptions() *opt.WriteOptions { - wo := &opt.WriteOptions{} - // request fsync on write for safety - wo.Sync = true - return wo -} - -func defaultReadOptions() *opt.ReadOptions { - ro := &opt.ReadOptions{} - return ro -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/writer.go b/vendor/github.com/blevesearch/bleve/index/store/goleveldb/writer.go deleted file mode 100644 index ce1d03c9..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/goleveldb/writer.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package goleveldb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/syndtr/goleveldb/leveldb" -) - -type Writer struct { - store *Store -} - -func newWriter(store *Store) (*Writer, error) { - store.writer.Lock() - return &Writer{ - store: store, - }, nil -} - -func (w *Writer) BytesSafeAfterClose() bool { - return false -} - -func (w *Writer) Set(key, val []byte) error { - return w.store.setlocked(key, val) -} - -func (w *Writer) Delete(key []byte) error { - return w.store.deletelocked(key) -} - -func (w *Writer) NewBatch() store.KVBatch { - rv := Batch{ - w: w, - merge: store.NewEmulatedMerge(w.store.mo), - batch: new(leveldb.Batch), - } - return &rv -} - -func (w *Writer) Close() error { - w.store.writer.Unlock() - return nil -} - -// these two methods can safely read using the regular -// methods without a read transaction, because we know -// that no one else is writing but us -func (w *Writer) Get(key []byte) ([]byte, error) { - return w.store.get(key) -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.store.iterator(key) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/batch.go b/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/batch.go deleted file mode 100644 index 3d80ece8..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/batch.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package rocksdb - -import ( - "github.com/tecbot/gorocksdb" -) - -type Batch struct { - w *Writer - batch *gorocksdb.WriteBatch -} - -func (b *Batch) Set(key, val []byte) { - b.batch.Put(key, val) -} - -func (b *Batch) Delete(key []byte) { - b.batch.Delete(key) -} - -func (b *Batch) Merge(key, val []byte) { - b.batch.Merge(key, val) -} - -func (b *Batch) Execute() error { - wopts := defaultWriteOptions() - err := b.w.store.db.Write(wopts, b.batch) - return err -} - -func (b *Batch) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/iterator.go deleted file mode 100644 index 9acdb76d..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/iterator.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package rocksdb - -import ( - "github.com/tecbot/gorocksdb" -) - -type Iterator struct { - store *Store - iterator *gorocksdb.Iterator -} - -func newIterator(store *Store) *Iterator { - ropts := defaultReadOptions() - rv := Iterator{ - store: store, - iterator: store.db.NewIterator(ropts), - } - return &rv -} - -func newIteratorWithSnapshot(store *Store, snapshot *gorocksdb.Snapshot) *Iterator { - options := defaultReadOptions() - options.SetSnapshot(snapshot) - rv := Iterator{ - store: store, - iterator: store.db.NewIterator(options), - } - return &rv -} - -func (ldi *Iterator) SeekFirst() { - ldi.iterator.SeekToFirst() -} - -func (ldi *Iterator) Seek(key []byte) { - ldi.iterator.Seek(key) -} - -func (ldi *Iterator) Next() { - ldi.iterator.Next() -} - -func (ldi *Iterator) Current() ([]byte, []byte, bool) { - if ldi.Valid() { - return ldi.Key(), ldi.Value(), true - } - return nil, nil, false -} - -func (ldi *Iterator) Key() []byte { - return ldi.iterator.Key().Data() -} - -func (ldi *Iterator) Value() []byte { - return ldi.iterator.Value().Data() -} - -func (ldi *Iterator) Valid() bool { - return ldi.iterator.Valid() -} - -func (ldi *Iterator) Close() error { - ldi.iterator.Close() - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/reader.go b/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/reader.go deleted file mode 100644 index 72a2e66a..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/reader.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package rocksdb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/tecbot/gorocksdb" -) - -type Reader struct { - store *Store - snapshot *gorocksdb.Snapshot -} - -func newReader(store *Store) (*Reader, error) { - return &Reader{ - store: store, - snapshot: store.db.NewSnapshot(), - }, nil -} - -func (r *Reader) BytesSafeAfterClose() bool { - return false -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - return r.store.getWithSnapshot(key, r.snapshot) -} - -func (r *Reader) Iterator(key []byte) store.KVIterator { - rv := newIteratorWithSnapshot(r.store, r.snapshot) - rv.Seek(key) - return rv -} - -func (r *Reader) Close() error { - r.snapshot.Release() - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store.go b/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store.go deleted file mode 100644 index c40899d1..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package rocksdb - -import ( - "fmt" - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - "github.com/tecbot/gorocksdb" -) - -const Name = "rocksdb" - -type Store struct { - path string - opts *gorocksdb.Options - db *gorocksdb.DB - writer sync.Mutex -} - -func New(path string, config map[string]interface{}) (*Store, error) { - rv := Store{ - path: path, - opts: gorocksdb.NewDefaultOptions(), - } - - _, err := applyConfig(rv.opts, config) - if err != nil { - return nil, err - } - - return &rv, nil -} - -func (ldbs *Store) Open() error { - var err error - ldbs.db, err = gorocksdb.OpenDb(ldbs.opts, ldbs.path) - if err != nil { - return err - } - return nil -} - -func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) { - ldbs.opts.SetMergeOperator(mo) -} - -func (ldbs *Store) get(key []byte) ([]byte, error) { - options := defaultReadOptions() - b, err := ldbs.db.Get(options, key) - return b.Data(), err -} - -func (ldbs *Store) getWithSnapshot(key []byte, snapshot *gorocksdb.Snapshot) ([]byte, error) { - options := defaultReadOptions() - options.SetSnapshot(snapshot) - b, err := ldbs.db.Get(options, key) - return b.Data(), err -} - -func (ldbs *Store) set(key, val []byte) error { - ldbs.writer.Lock() - defer ldbs.writer.Unlock() - return ldbs.setlocked(key, val) -} - -func (ldbs *Store) setlocked(key, val []byte) error { - options := defaultWriteOptions() - err := ldbs.db.Put(options, key, val) - return err -} - -func (ldbs *Store) delete(key []byte) error { - ldbs.writer.Lock() - defer ldbs.writer.Unlock() - return ldbs.deletelocked(key) -} - -func (ldbs *Store) deletelocked(key []byte) error { - options := defaultWriteOptions() - err := ldbs.db.Delete(options, key) - return err -} - -func (ldbs *Store) Close() error { - ldbs.db.Close() - return nil -} - -func (ldbs *Store) iterator(key []byte) store.KVIterator { - rv := newIterator(ldbs) - rv.Seek(key) - return rv -} - -func (ldbs *Store) Reader() (store.KVReader, error) { - return newReader(ldbs) -} - -func (ldbs *Store) Writer() (store.KVWriter, error) { - return newWriter(ldbs) -} - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") - } - return New(path, config) -} - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} - -func applyConfig(o *gorocksdb.Options, config map[string]interface{}) ( - *gorocksdb.Options, error) { - - cim, ok := config["create_if_missing"].(bool) - if ok { - o.SetCreateIfMissing(cim) - } - - eie, ok := config["error_if_exists"].(bool) - if ok { - o.SetErrorIfExists(eie) - } - - wbs, ok := config["write_buffer_size"].(float64) - if ok { - o.SetWriteBufferSize(int(wbs)) - } - - return o, nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store_test.go b/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store_test.go deleted file mode 100644 index 246cc09d..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store_test.go +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package rocksdb - -import ( - "os" - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -var rocksdbTestOptions = map[string]interface{}{ - "create_if_missing": true, -} - -func TestGoRocksDBStore(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s, err := New("test", rocksdbTestOptions) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestKVStore(t, s) -} - -func TestReaderIsolation(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s, err := New("test", rocksdbTestOptions) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestReaderIsolation(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify that we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got nil") - } - - // ensure that the director iterator sees it - count = 0 - it2 := newReader.Iterator([]byte{0}) - defer func() { - err := it2.Close() - if err != nil { - t.Fatal(err) - } - }() - for it2.Valid() { - it2.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it3 := reader.Iterator([]byte{0}) - defer func() { - err := it3.Close() - if err != nil { - t.Fatal(err) - } - }() - for it3.Valid() { - it3.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/util.go b/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/util.go deleted file mode 100644 index 830fe490..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/util.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package rocksdb - -import ( - "github.com/tecbot/gorocksdb" -) - -func defaultWriteOptions() *gorocksdb.WriteOptions { - wo := gorocksdb.NewDefaultWriteOptions() - // request fsync on write for safety - wo.SetSync(true) - return wo -} - -func defaultReadOptions() *gorocksdb.ReadOptions { - ro := gorocksdb.NewDefaultReadOptions() - return ro -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/writer.go b/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/writer.go deleted file mode 100644 index 2c971c7b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gorocksdb/writer.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package rocksdb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/tecbot/gorocksdb" -) - -type Writer struct { - store *Store -} - -func newWriter(store *Store) (*Writer, error) { - store.writer.Lock() - return &Writer{ - store: store, - }, nil -} - -func (w *Writer) BytesSafeAfterClose() bool { - return false -} - -func (w *Writer) Set(key, val []byte) error { - return w.store.setlocked(key, val) -} - -func (w *Writer) Delete(key []byte) error { - return w.store.deletelocked(key) -} - -func (w *Writer) NewBatch() store.KVBatch { - rv := Batch{ - w: w, - batch: gorocksdb.NewWriteBatch(), - } - return &rv -} - -func (w *Writer) Close() error { - w.store.writer.Unlock() - return nil -} - -// these two methods can safely read using the regular -// methods without a read transaction, because we know -// that no one else is writing but us -func (w *Writer) Get(key []byte) ([]byte, error) { - return w.store.get(key) -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.store.iterator(key) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap.go deleted file mode 100644 index e6f1eb65..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -// Package gtreap provides an in-memory implementation of the -// KVStore interfaces using the gtreap balanced-binary treap, -// copy-on-write data structure. -package gtreap - -import ( - "bytes" - "fmt" - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - - "github.com/steveyen/gtreap" -) - -const Name = "gtreap" - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} - -const MAX_CONCURRENT_WRITERS = 1 - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - s := &Store{ - availableWriters: make(chan bool, MAX_CONCURRENT_WRITERS), - t: gtreap.NewTreap(itemCompare), - } - for i := 0; i < MAX_CONCURRENT_WRITERS; i++ { - s.availableWriters <- true - } - return s, nil -} - -type Item struct { - k []byte - v []byte -} - -func itemCompare(a, b interface{}) int { - return bytes.Compare(a.(*Item).k, b.(*Item).k) -} - -type Store struct { - availableWriters chan bool - - m sync.Mutex - t *gtreap.Treap - - mo store.MergeOperator -} - -type Writer struct { - s *Store -} - -func (s *Store) Open() error { - return nil -} - -func (s *Store) SetMergeOperator(mo store.MergeOperator) { - s.mo = mo -} - -func (s *Store) Close() error { - close(s.availableWriters) - return nil -} - -func (s *Store) Reader() (store.KVReader, error) { - s.m.Lock() - t := s.t - s.m.Unlock() - return &Reader{t: t}, nil -} - -func (s *Store) Writer() (store.KVWriter, error) { - available, ok := <-s.availableWriters - if !ok || !available { - return nil, fmt.Errorf("no available writers") - } - - return &Writer{s: s}, nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap_test.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap_test.go deleted file mode 100644 index 19b568d1..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap_test.go +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -package gtreap - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -func TestGTreapStore(t *testing.T) { - s, err := StoreConstructor(nil) - if err != nil { - t.Fatal(err) - } - - CommonTestKVStore(t, s) -} - -func TestReaderIsolation(t *testing.T) { - s, err := StoreConstructor(nil) - if err != nil { - t.Fatal(err) - } - - CommonTestReaderIsolation(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify that we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - valB := []byte("val-b") - err = writer.Set([]byte("b"), valB) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got %s", val) - } - - // ensure that the director iterator sees it - count = 0 - it2 := newReader.Iterator([]byte{0}) - defer func() { - err := it2.Close() - if err != nil { - t.Fatal(err) - } - }() - for it2.Valid() { - it2.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it3 := reader.Iterator([]byte{0}) - defer func() { - err := it3.Close() - if err != nil { - t.Fatal(err) - } - }() - for it3.Valid() { - it3.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go index 3bed1a2b..092ccf24 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go +++ b/vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go @@ -1,13 +1,16 @@ // Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Package gtreap provides an in-memory implementation of the // KVStore interfaces using the gtreap balanced-binary treap, @@ -15,6 +18,7 @@ package gtreap import ( + "bytes" "sync" "github.com/steveyen/gtreap" @@ -28,22 +32,33 @@ type Iterator struct { nextCh chan *Item curr *Item currOk bool -} -func newIterator(t *gtreap.Treap) *Iterator { - return &Iterator{t: t} -} - -func (w *Iterator) SeekFirst() { - min := w.t.Min() - if min != nil { - w.restart(min.(*Item)) - } else { - w.restart(nil) - } + prefix []byte + start []byte + end []byte } func (w *Iterator) Seek(k []byte) { + if w.start != nil && bytes.Compare(k, w.start) < 0 { + k = w.start + } + if w.prefix != nil && !bytes.HasPrefix(k, w.prefix) { + if bytes.Compare(k, w.prefix) < 0 { + k = w.prefix + } else { + var end []byte + for i := len(w.prefix) - 1; i >= 0; i-- { + c := w.prefix[i] + if c < 0xff { + end = make([]byte, i+1) + copy(end, w.prefix) + end[i] = c + 1 + break + } + } + k = end + } + } w.restart(&Item{k: k}) } @@ -93,6 +108,11 @@ func (w *Iterator) Current() ([]byte, []byte, bool) { if !w.currOk || w.curr == nil { return nil, nil, false } + if w.prefix != nil && !bytes.HasPrefix(w.curr.k, w.prefix) { + return nil, nil, false + } else if w.end != nil && bytes.Compare(w.curr.k, w.end) >= 0 { + return nil, nil, false + } return w.curr.k, w.curr.v, w.currOk } diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go index 4a18a452..98254d34 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go +++ b/vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go @@ -1,13 +1,16 @@ // Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Package gtreap provides an in-memory implementation of the // KVStore interfaces using the gtreap balanced-binary treap, @@ -24,20 +27,38 @@ type Reader struct { t *gtreap.Treap } -func (w *Reader) BytesSafeAfterClose() bool { - return false -} - func (w *Reader) Get(k []byte) (v []byte, err error) { + var rv []byte itm := w.t.Get(&Item{k: k}) if itm != nil { - return itm.(*Item).v, nil + rv = make([]byte, len(itm.(*Item).v)) + copy(rv, itm.(*Item).v) + return rv, nil } return nil, nil } -func (w *Reader) Iterator(k []byte) store.KVIterator { - return newIterator(w.t).restart(&Item{k: k}) +func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) { + return store.MultiGet(r, keys) +} + +func (w *Reader) PrefixIterator(k []byte) store.KVIterator { + rv := Iterator{ + t: w.t, + prefix: k, + } + rv.restart(&Item{k: k}) + return &rv +} + +func (w *Reader) RangeIterator(start, end []byte) store.KVIterator { + rv := Iterator{ + t: w.t, + start: start, + end: end, + } + rv.restart(&Item{k: start}) + return &rv } func (w *Reader) Close() error { diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go new file mode 100644 index 00000000..3e6c5fec --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go @@ -0,0 +1,82 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package gtreap provides an in-memory implementation of the +// KVStore interfaces using the gtreap balanced-binary treap, +// copy-on-write data structure. + +package gtreap + +import ( + "bytes" + "fmt" + "os" + "sync" + + "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/registry" + "github.com/steveyen/gtreap" +) + +const Name = "gtreap" + +type Store struct { + m sync.Mutex + t *gtreap.Treap + mo store.MergeOperator +} + +type Item struct { + k []byte + v []byte +} + +func itemCompare(a, b interface{}) int { + return bytes.Compare(a.(*Item).k, b.(*Item).k) +} + +func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path != "" { + return nil, os.ErrInvalid + } + + rv := Store{ + t: gtreap.NewTreap(itemCompare), + mo: mo, + } + return &rv, nil +} + +func (s *Store) Close() error { + return nil +} + +func (s *Store) Reader() (store.KVReader, error) { + s.m.Lock() + t := s.t + s.m.Unlock() + return &Reader{t: t}, nil +} + +func (s *Store) Writer() (store.KVWriter, error) { + return &Writer{s: s}, nil +} + +func init() { + registry.RegisterKVStore(Name, New) +} diff --git a/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go b/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go index aa95256c..777aab40 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go +++ b/vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go @@ -1,13 +1,16 @@ // Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Package gtreap provides an in-memory implementation of the // KVStore interfaces using the gtreap balanced-binary treap, @@ -15,58 +18,59 @@ package gtreap import ( + "fmt" "math/rand" "github.com/blevesearch/bleve/index/store" ) -func (w *Writer) BytesSafeAfterClose() bool { - return false -} - -func (w *Writer) Get(k []byte) (v []byte, err error) { - w.s.m.Lock() - t := w.s.t - w.s.m.Unlock() - - itm := t.Get(&Item{k: k}) - if itm != nil { - return itm.(*Item).v, nil - } - return nil, nil -} - -func (w *Writer) Iterator(k []byte) store.KVIterator { - w.s.m.Lock() - t := w.s.t - w.s.m.Unlock() - - return newIterator(t).restart(&Item{k: k}) -} - -func (w *Writer) Close() error { - w.s.availableWriters <- true - w.s = nil - - return nil -} - -func (w *Writer) Set(k, v []byte) (err error) { - w.s.m.Lock() - w.s.t = w.s.t.Upsert(&Item{k: k, v: v}, rand.Int()) - w.s.m.Unlock() - - return nil -} - -func (w *Writer) Delete(k []byte) (err error) { - w.s.m.Lock() - w.s.t = w.s.t.Delete(&Item{k: k}) - w.s.m.Unlock() - - return nil +type Writer struct { + s *Store } func (w *Writer) NewBatch() store.KVBatch { - return store.NewEmulatedBatch(w, w.s.mo) + return store.NewEmulatedBatch(w.s.mo) +} + +func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) { + return make([]byte, options.TotalBytes), w.NewBatch(), nil +} + +func (w *Writer) ExecuteBatch(batch store.KVBatch) error { + + emulatedBatch, ok := batch.(*store.EmulatedBatch) + if !ok { + return fmt.Errorf("wrong type of batch") + } + + w.s.m.Lock() + for k, mergeOps := range emulatedBatch.Merger.Merges { + kb := []byte(k) + var existingVal []byte + existingItem := w.s.t.Get(&Item{k: kb}) + if existingItem != nil { + existingVal = w.s.t.Get(&Item{k: kb}).(*Item).v + } + mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps) + if !fullMergeOk { + return fmt.Errorf("merge operator returned failure") + } + w.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int()) + } + + for _, op := range emulatedBatch.Ops { + if op.V != nil { + w.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int()) + } else { + w.s.t = w.s.t.Delete(&Item{k: op.K}) + } + } + w.s.m.Unlock() + + return nil +} + +func (w *Writer) Close() error { + w.s = nil + return nil } diff --git a/vendor/github.com/blevesearch/bleve/index/store/inmem/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/inmem/iterator.go deleted file mode 100644 index 042bd890..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/inmem/iterator.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package inmem - -import ( - "github.com/ryszard/goskiplist/skiplist" -) - -type Iterator struct { - store *Store - iterator skiplist.Iterator - valid bool -} - -func newIterator(store *Store) *Iterator { - rv := Iterator{ - store: store, - iterator: store.list.Iterator(), - } - return &rv -} - -func (i *Iterator) SeekFirst() { - i.Seek([]byte{0}) -} - -func (i *Iterator) Seek(k []byte) { - i.valid = i.iterator.Seek(string(k)) -} - -func (i *Iterator) Next() { - i.valid = i.iterator.Next() -} - -func (i *Iterator) Current() ([]byte, []byte, bool) { - if i.valid { - return []byte(i.Key()), []byte(i.Value()), true - } - return nil, nil, false -} - -func (i *Iterator) Key() []byte { - if i.valid { - return []byte(i.iterator.Key().(string)) - } - return nil -} - -func (i *Iterator) Value() []byte { - if i.valid { - return []byte(i.iterator.Value().(string)) - } - return nil -} - -func (i *Iterator) Valid() bool { - return i.valid -} - -func (i *Iterator) Close() error { - i.iterator.Close() - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/inmem/reader.go b/vendor/github.com/blevesearch/bleve/index/store/inmem/reader.go deleted file mode 100644 index d9966277..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/inmem/reader.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package inmem - -import ( - "github.com/blevesearch/bleve/index/store" -) - -type Reader struct { - store *Store -} - -func newReader(store *Store) (*Reader, error) { - return &Reader{ - store: store, - }, nil -} - -func (r *Reader) BytesSafeAfterClose() bool { - return false -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - return r.store.get(key) -} - -func (r *Reader) Iterator(key []byte) store.KVIterator { - return r.store.iterator(key) -} - -func (r *Reader) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/inmem/store.go b/vendor/github.com/blevesearch/bleve/index/store/inmem/store.go deleted file mode 100644 index d121de59..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/inmem/store.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package inmem - -import ( - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - "github.com/ryszard/goskiplist/skiplist" -) - -const Name = "mem" - -type Store struct { - list *skiplist.SkipList - writer sync.Mutex - mo store.MergeOperator -} - -func New() (*Store, error) { - rv := Store{ - list: skiplist.NewStringMap(), - } - - return &rv, nil -} - -func MustOpen() *Store { - rv := Store{ - list: skiplist.NewStringMap(), - } - - return &rv -} - -func (i *Store) Open() error { - return nil -} - -func (i *Store) SetMergeOperator(mo store.MergeOperator) { - i.mo = mo -} - -func (i *Store) get(key []byte) ([]byte, error) { - val, ok := i.list.Get(string(key)) - if ok { - return []byte(val.(string)), nil - } - return nil, nil -} - -func (i *Store) set(key, val []byte) error { - i.writer.Lock() - defer i.writer.Unlock() - return i.setlocked(key, val) -} - -func (i *Store) setlocked(key, val []byte) error { - i.list.Set(string(key), string(val)) - return nil -} - -func (i *Store) delete(key []byte) error { - i.writer.Lock() - defer i.writer.Unlock() - return i.deletelocked(key) -} - -func (i *Store) deletelocked(key []byte) error { - i.list.Delete(string(key)) - return nil -} - -func (i *Store) Close() error { - return nil -} - -func (i *Store) iterator(key []byte) store.KVIterator { - rv := newIterator(i) - rv.Seek(key) - return rv -} - -func (i *Store) Reader() (store.KVReader, error) { - return newReader(i) -} - -func (i *Store) Writer() (store.KVWriter, error) { - return newWriter(i) -} - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - return New() -} - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/inmem/store_test.go b/vendor/github.com/blevesearch/bleve/index/store/inmem/store_test.go deleted file mode 100644 index 39079459..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/inmem/store_test.go +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package inmem - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -func TestStore(t *testing.T) { - s, err := New() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestKVStore(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify that we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got nil") - } - - // ensure that the director iterator sees it - count = 0 - it = newReader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it = reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/inmem/writer.go b/vendor/github.com/blevesearch/bleve/index/store/inmem/writer.go deleted file mode 100644 index f88c4e01..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/inmem/writer.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package inmem - -import ( - "github.com/blevesearch/bleve/index/store" -) - -type Writer struct { - store *Store -} - -func newWriter(store *Store) (*Writer, error) { - store.writer.Lock() - return &Writer{ - store: store, - }, nil -} - -func (w *Writer) BytesSafeAfterClose() bool { - return false -} - -func (w *Writer) Set(key, val []byte) error { - return w.store.setlocked(key, val) -} - -func (w *Writer) Delete(key []byte) error { - return w.store.deletelocked(key) -} - -func (w *Writer) NewBatch() store.KVBatch { - return store.NewEmulatedBatch(w, w.store.mo) -} - -func (w *Writer) Close() error { - w.store.writer.Unlock() - return nil -} - -// these two methods can safely read using the regular -// methods without a read transaction, because we know -// that no one else is writing but us -func (w *Writer) Get(key []byte) ([]byte, error) { - return w.store.get(key) -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.store.iterator(key) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/kvstore.go b/vendor/github.com/blevesearch/bleve/index/store/kvstore.go index 4be8e0a5..34698c7b 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/kvstore.go +++ b/vendor/github.com/blevesearch/bleve/index/store/kvstore.go @@ -1,53 +1,174 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package store -type KVBatch interface { - Set(key, val []byte) - Delete(key []byte) - Merge(key, val []byte) - Execute() error +import "encoding/json" + +// KVStore is an abstraction for working with KV stores. Note that +// in order to be used with the bleve.registry, it must also implement +// a constructor function of the registry.KVStoreConstructor type. +type KVStore interface { + + // Writer returns a KVWriter which can be used to + // make changes to the KVStore. If a writer cannot + // be obtained a non-nil error is returned. + Writer() (KVWriter, error) + + // Reader returns a KVReader which can be used to + // read data from the KVStore. If a reader cannot + // be obtained a non-nil error is returned. + Reader() (KVReader, error) + + // Close closes the KVStore Close() error } +// KVReader is an abstraction of an **ISOLATED** reader +// In this context isolated is defined to mean that +// writes/deletes made after the KVReader is opened +// are not observed. +// Because there is usually a cost associated with +// keeping isolated readers active, users should +// close them as soon as they are no longer needed. +type KVReader interface { + + // Get returns the value associated with the key + // If the key does not exist, nil is returned. + // The caller owns the bytes returned. + Get(key []byte) ([]byte, error) + + // MultiGet retrieves multiple values in one call. + MultiGet(keys [][]byte) ([][]byte, error) + + // PrefixIterator returns a KVIterator that will + // visit all K/V pairs with the provided prefix + PrefixIterator(prefix []byte) KVIterator + + // RangeIterator returns a KVIterator that will + // visit all K/V pairs >= start AND < end + RangeIterator(start, end []byte) KVIterator + + // Close closes the iterator + Close() error +} + +// KVIterator is an abstraction around key iteration type KVIterator interface { - SeekFirst() - Seek([]byte) + + // Seek will advance the iterator to the specified key + Seek(key []byte) + + // Next will advance the iterator to the next key Next() - Current() ([]byte, []byte, bool) + // Key returns the key pointed to by the iterator + // The bytes returned are **ONLY** valid until the next call to Seek/Next/Close + // Continued use after that requires that they be copied. Key() []byte + + // Value returns the value pointed to by the iterator + // The bytes returned are **ONLY** valid until the next call to Seek/Next/Close + // Continued use after that requires that they be copied. Value() []byte + + // Valid returns whether or not the iterator is in a valid state Valid() bool + // Current returns Key(),Value(),Valid() in a single operation + Current() ([]byte, []byte, bool) + + // Close closes the iterator Close() error } -type KVStore interface { - Open() error - SetMergeOperator(MergeOperator) - Writer() (KVWriter, error) - Reader() (KVReader, error) - Close() error -} - +// KVWriter is an abstraction for mutating the KVStore +// KVWriter does **NOT** enforce restrictions of a single writer +// if the underlying KVStore allows concurrent writes, the +// KVWriter interface should also do so, it is up to the caller +// to do this in a way that is safe and makes sense type KVWriter interface { - KVReader - Set(key, val []byte) error - Delete(key []byte) error - NewBatch() KVBatch -} -type KVReader interface { - BytesSafeAfterClose() bool - Get(key []byte) ([]byte, error) - Iterator(key []byte) KVIterator + // NewBatch returns a KVBatch for performing batch operations on this kvstore + NewBatch() KVBatch + + // NewBatchEx returns a KVBatch and an associated byte array + // that's pre-sized based on the KVBatchOptions. The caller can + // use the returned byte array for keys and values associated with + // the batch. Once the batch is either executed or closed, the + // associated byte array should no longer be accessed by the + // caller. + NewBatchEx(KVBatchOptions) ([]byte, KVBatch, error) + + // ExecuteBatch will execute the KVBatch, the provided KVBatch **MUST** have + // been created by the same KVStore (though not necessarily the same KVWriter) + // Batch execution is atomic, either all the operations or none will be performed + ExecuteBatch(batch KVBatch) error + + // Close closes the writer Close() error } + +// KVBatchOptions provides the KVWriter.NewBatchEx() method with batch +// preparation and preallocation information. +type KVBatchOptions struct { + // TotalBytes is the sum of key and value bytes needed by the + // caller for the entire batch. It affects the size of the + // returned byte array of KVWrite.NewBatchEx(). + TotalBytes int + + // NumSets is the number of Set() calls the caller will invoke on + // the KVBatch. + NumSets int + + // NumDeletes is the number of Delete() calls the caller will invoke + // on the KVBatch. + NumDeletes int + + // NumMerges is the number of Merge() calls the caller will invoke + // on the KVBatch. + NumMerges int +} + +// KVBatch is an abstraction for making multiple KV mutations at once +type KVBatch interface { + + // Set updates the key with the specified value + // both key and value []byte may be reused as soon as this call returns + Set(key, val []byte) + + // Delete removes the specified key + // the key []byte may be reused as soon as this call returns + Delete(key []byte) + + // Merge merges old value with the new value at the specified key + // as prescribed by the KVStores merge operator + // both key and value []byte may be reused as soon as this call returns + Merge(key, val []byte) + + // Reset frees resources for this batch and allows reuse + Reset() + + // Close frees resources + Close() error +} + +// KVStoreStats is an optional interface that KVStores can implement +// if they're able to report any useful stats +type KVStoreStats interface { + // Stats returns a JSON serializable object representing stats for this KVStore + Stats() json.Marshaler + + StatsMap() map[string]interface{} +} diff --git a/vendor/github.com/blevesearch/bleve/index/store/leveldb/batch.go b/vendor/github.com/blevesearch/bleve/index/store/leveldb/batch.go deleted file mode 100644 index 2e95286a..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/leveldb/batch.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package leveldb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/jmhodges/levigo" -) - -type Batch struct { - w *Writer - merge *store.EmulatedMerge - batch *levigo.WriteBatch -} - -func (b *Batch) Set(key, val []byte) { - b.batch.Put(key, val) -} - -func (b *Batch) Delete(key []byte) { - b.batch.Delete(key) -} - -func (b *Batch) Merge(key, val []byte) { - b.merge.Merge(key, val) -} - -func (b *Batch) Execute() error { - // first process merges - ops, err := b.merge.ExecuteDeferred(b.w) - if err != nil { - return err - } - for _, op := range ops { - b.batch.Put(op.K, op.V) - } - - wopts := defaultWriteOptions() - defer wopts.Close() - err = b.w.store.db.Write(wopts, b.batch) - return err -} - -func (b *Batch) Close() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/leveldb/iterator.go b/vendor/github.com/blevesearch/bleve/index/store/leveldb/iterator.go deleted file mode 100644 index 10ef8d7d..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/leveldb/iterator.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package leveldb - -import ( - "github.com/jmhodges/levigo" -) - -type Iterator struct { - store *Store - iterator *levigo.Iterator -} - -func newIterator(store *Store) *Iterator { - ropts := defaultReadOptions() - rv := Iterator{ - store: store, - iterator: store.db.NewIterator(ropts), - } - ropts.Close() - return &rv -} - -func newIteratorWithSnapshot(store *Store, snapshot *levigo.Snapshot) *Iterator { - options := defaultReadOptions() - options.SetSnapshot(snapshot) - rv := Iterator{ - store: store, - iterator: store.db.NewIterator(options), - } - options.Close() - return &rv -} - -func (ldi *Iterator) SeekFirst() { - ldi.iterator.SeekToFirst() -} - -func (ldi *Iterator) Seek(key []byte) { - ldi.iterator.Seek(key) -} - -func (ldi *Iterator) Next() { - ldi.iterator.Next() -} - -func (ldi *Iterator) Current() ([]byte, []byte, bool) { - if ldi.Valid() { - return ldi.Key(), ldi.Value(), true - } - return nil, nil, false -} - -func (ldi *Iterator) Key() []byte { - return ldi.iterator.Key() -} - -func (ldi *Iterator) Value() []byte { - return ldi.iterator.Value() -} - -func (ldi *Iterator) Valid() bool { - return ldi.iterator.Valid() -} - -func (ldi *Iterator) Close() error { - ldi.iterator.Close() - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/leveldb/reader.go b/vendor/github.com/blevesearch/bleve/index/store/leveldb/reader.go deleted file mode 100644 index c6360317..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/leveldb/reader.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package leveldb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/jmhodges/levigo" -) - -type Reader struct { - store *Store - snapshot *levigo.Snapshot -} - -func newReader(store *Store) (*Reader, error) { - return &Reader{ - store: store, - snapshot: store.db.NewSnapshot(), - }, nil -} - -func (r *Reader) BytesSafeAfterClose() bool { - return true -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - return r.store.getWithSnapshot(key, r.snapshot) -} - -func (r *Reader) Iterator(key []byte) store.KVIterator { - rv := newIteratorWithSnapshot(r.store, r.snapshot) - rv.Seek(key) - return rv -} - -func (r *Reader) Close() error { - r.store.db.ReleaseSnapshot(r.snapshot) - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/leveldb/store.go b/vendor/github.com/blevesearch/bleve/index/store/leveldb/store.go deleted file mode 100644 index 46d9a3d7..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/leveldb/store.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package leveldb - -import ( - "fmt" - "sync" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - "github.com/jmhodges/levigo" -) - -const Name = "leveldb" - -type Store struct { - path string - opts *levigo.Options - db *levigo.DB - writer sync.Mutex - mo store.MergeOperator -} - -func New(path string, config map[string]interface{}) (*Store, error) { - rv := Store{ - path: path, - opts: levigo.NewOptions(), - } - - _, err := applyConfig(rv.opts, config) - if err != nil { - return nil, err - } - - return &rv, nil -} - -func (ldbs *Store) Open() error { - var err error - ldbs.db, err = levigo.Open(ldbs.path, ldbs.opts) - if err != nil { - return err - } - return nil -} - -func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) { - ldbs.mo = mo -} - -func (ldbs *Store) get(key []byte) ([]byte, error) { - options := defaultReadOptions() - b, err := ldbs.db.Get(options, key) - options.Close() - return b, err -} - -func (ldbs *Store) getWithSnapshot(key []byte, snapshot *levigo.Snapshot) ([]byte, error) { - options := defaultReadOptions() - options.SetSnapshot(snapshot) - b, err := ldbs.db.Get(options, key) - options.Close() - return b, err -} - -func (ldbs *Store) set(key, val []byte) error { - ldbs.writer.Lock() - defer ldbs.writer.Unlock() - return ldbs.setlocked(key, val) -} - -func (ldbs *Store) setlocked(key, val []byte) error { - options := defaultWriteOptions() - err := ldbs.db.Put(options, key, val) - options.Close() - return err -} - -func (ldbs *Store) delete(key []byte) error { - ldbs.writer.Lock() - defer ldbs.writer.Unlock() - return ldbs.deletelocked(key) -} - -func (ldbs *Store) deletelocked(key []byte) error { - options := defaultWriteOptions() - err := ldbs.db.Delete(options, key) - options.Close() - return err -} - -func (ldbs *Store) Close() error { - ldbs.db.Close() - ldbs.opts.Close() - return nil -} - -func (ldbs *Store) iterator(key []byte) store.KVIterator { - rv := newIterator(ldbs) - rv.Seek(key) - return rv -} - -func (ldbs *Store) Reader() (store.KVReader, error) { - return newReader(ldbs) -} - -func (ldbs *Store) Writer() (store.KVWriter, error) { - return newWriter(ldbs) -} - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - path, ok := config["path"].(string) - if !ok { - return nil, fmt.Errorf("must specify path") - } - return New(path, config) -} - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} - -func applyConfig(o *levigo.Options, config map[string]interface{}) ( - *levigo.Options, error) { - - cim, ok := config["create_if_missing"].(bool) - if ok { - o.SetCreateIfMissing(cim) - } - - eie, ok := config["error_if_exists"].(bool) - if ok { - o.SetErrorIfExists(eie) - } - - wbs, ok := config["write_buffer_size"].(float64) - if ok { - o.SetWriteBufferSize(int(wbs)) - } - - bs, ok := config["block_size"].(float64) - if ok { - o.SetBlockSize(int(bs)) - } - - bri, ok := config["block_restart_interval"].(float64) - if ok { - o.SetBlockRestartInterval(int(bri)) - } - - lcc, ok := config["lru_cache_capacity"].(float64) - if ok { - lruCache := levigo.NewLRUCache(int(lcc)) - o.SetCache(lruCache) - } - - bfbpk, ok := config["bloom_filter_bits_per_key"].(float64) - if ok { - bf := levigo.NewBloomFilter(int(bfbpk)) - o.SetFilterPolicy(bf) - } - - return o, nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/leveldb/store_test.go b/vendor/github.com/blevesearch/bleve/index/store/leveldb/store_test.go deleted file mode 100644 index 3eca69c1..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/leveldb/store_test.go +++ /dev/null @@ -1,298 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package leveldb - -import ( - "os" - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -var leveldbTestOptions = map[string]interface{}{ - "create_if_missing": true, -} - -func TestLevelDBStore(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s, err := New("test", leveldbTestOptions) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestKVStore(t, s) -} - -func TestReaderIsolation(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s, err := New("test", leveldbTestOptions) - if err != nil { - t.Fatal(err) - } - err = s.Open() - if err != nil { - t.Fatal(err) - } - defer func() { - err := s.Close() - if err != nil { - t.Fatal(err) - } - }() - - CommonTestReaderIsolation(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify that we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got nil") - } - - // ensure that the director iterator sees it - count = 0 - it2 := newReader.Iterator([]byte{0}) - defer func() { - err := it2.Close() - if err != nil { - t.Fatal(err) - } - }() - for it2.Valid() { - it2.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it3 := reader.Iterator([]byte{0}) - defer func() { - err := it3.Close() - if err != nil { - t.Fatal(err) - } - }() - for it3.Valid() { - it3.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/leveldb/util.go b/vendor/github.com/blevesearch/bleve/index/store/leveldb/util.go deleted file mode 100644 index a34f6f84..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/leveldb/util.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package leveldb - -import ( - "github.com/jmhodges/levigo" -) - -func defaultWriteOptions() *levigo.WriteOptions { - wo := levigo.NewWriteOptions() - // request fsync on write for safety - wo.SetSync(true) - return wo -} - -func defaultReadOptions() *levigo.ReadOptions { - ro := levigo.NewReadOptions() - return ro -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/leveldb/writer.go b/vendor/github.com/blevesearch/bleve/index/store/leveldb/writer.go deleted file mode 100644 index 1ef3a4e3..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/leveldb/writer.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package leveldb - -import ( - "github.com/blevesearch/bleve/index/store" - "github.com/jmhodges/levigo" -) - -type Writer struct { - store *Store -} - -func newWriter(store *Store) (*Writer, error) { - store.writer.Lock() - return &Writer{ - store: store, - }, nil -} - -func (w *Writer) BytesSafeAfterClose() bool { - return true -} - -func (w *Writer) Set(key, val []byte) error { - return w.store.setlocked(key, val) -} - -func (w *Writer) Delete(key []byte) error { - return w.store.deletelocked(key) -} - -func (w *Writer) NewBatch() store.KVBatch { - rv := Batch{ - w: w, - merge: store.NewEmulatedMerge(w.store.mo), - batch: levigo.NewWriteBatch(), - } - return &rv -} - -func (w *Writer) Close() error { - w.store.writer.Unlock() - return nil -} - -// these two methods can safely read using the regular -// methods without a read transaction, because we know -// that no one else is writing but us -func (w *Writer) Get(key []byte) ([]byte, error) { - return w.store.get(key) -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.store.iterator(key) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/merge.go b/vendor/github.com/blevesearch/bleve/index/store/merge.go index 390727bf..ca2561b0 100644 --- a/vendor/github.com/blevesearch/bleve/index/store/merge.go +++ b/vendor/github.com/blevesearch/bleve/index/store/merge.go @@ -1,18 +1,19 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package store -import ( - "fmt" -) - // At the moment this happens to be the same interface as described by // RocksDB, but this may not always be the case. @@ -32,41 +33,20 @@ type MergeOperator interface { Name() string } -// EmulatedMergeSingle removes some duplicated code across -// KV stores which do not support merge operations -// on their own. It is up to the caller to ensure -// that an appropriate lock has been acquired in -// order for this behavior to be valid -func EmulatedMergeSingle(writer KVWriter, mo MergeOperator, key []byte, operand []byte) error { - existingValue, err := writer.Get(key) - if err != nil { - return err - } - newValue, ok := mo.FullMerge(key, existingValue, [][]byte{operand}) - if !ok { - return fmt.Errorf("merge operator returned failure") - } - err = writer.Set(key, newValue) - if err != nil { - return err - } - return nil -} - type EmulatedMerge struct { - merges map[string][][]byte + Merges map[string][][]byte mo MergeOperator } func NewEmulatedMerge(mo MergeOperator) *EmulatedMerge { return &EmulatedMerge{ - merges: make(map[string][][]byte), + Merges: make(map[string][][]byte), mo: mo, } } func (m *EmulatedMerge) Merge(key, val []byte) { - ops, ok := m.merges[string(key)] + ops, ok := m.Merges[string(key)] if ok && len(ops) > 0 { last := ops[len(ops)-1] mergedVal, partialMergeOk := m.mo.PartialMerge(key, last, val) @@ -80,41 +60,5 @@ func (m *EmulatedMerge) Merge(key, val []byte) { } else { ops = [][]byte{val} } - m.merges[string(key)] = ops -} - -func (m *EmulatedMerge) Execute(w KVWriter) error { - for k, mergeOps := range m.merges { - kb := []byte(k) - existingVal, err := w.Get(kb) - if err != nil { - return err - } - mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps) - if !fullMergeOk { - return fmt.Errorf("merge operator returned failure") - } - err = w.Set(kb, mergedVal) - if err != nil { - return err - } - } - return nil -} - -func (m *EmulatedMerge) ExecuteDeferred(w KVWriter) ([]*op, error) { - rv := make([]*op, 0, 1000) - for k, mergeOps := range m.merges { - kb := []byte(k) - existingVal, err := w.Get(kb) - if err != nil { - return nil, err - } - mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps) - if !fullMergeOk { - return nil, fmt.Errorf("merge operator returned failure") - } - rv = append(rv, &op{kb, mergedVal}) - } - return rv, nil + m.Merges[string(key)] = ops } diff --git a/vendor/github.com/blevesearch/bleve/index/store/metrics/metrics.go b/vendor/github.com/blevesearch/bleve/index/store/metrics/metrics.go deleted file mode 100644 index c5757d7a..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/metrics/metrics.go +++ /dev/null @@ -1,476 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -// +build debug - -// Package metrics provides a bleve.store.KVStore implementation that -// wraps another, real KVStore implementation, and uses go-metrics to -// track runtime performance metrics. -package metrics - -import ( - "container/list" - "encoding/json" - "fmt" - "io" - "sync" - "time" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" - - "github.com/rcrowley/go-metrics" -) - -const Name = "metrics" -const MaxErrors = 100 - -func init() { - registry.RegisterKVStore(Name, StoreConstructor) -} - -func StoreConstructor(config map[string]interface{}) (store.KVStore, error) { - name, ok := config["kvStoreName_actual"].(string) - if !ok || name == "" { - return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+ - " config: %#v", config) - } - - if name == Name { - return nil, fmt.Errorf("metrics: circular kvStoreName_actual") - } - - ctr := registry.KVStoreConstructorByName(name) - if ctr == nil { - return nil, fmt.Errorf("metrics: no kv store constructor,"+ - " kvStoreName_actual: %s", name) - } - - kvs, err := ctr(config) - if err != nil { - return nil, err - } - - return NewBleveMetricsStore(kvs), nil -} - -func NewBleveMetricsStore(o store.KVStore) *Store { - return &Store{ - o: o, - - TimerReaderGet: metrics.NewTimer(), - TimerReaderIterator: metrics.NewTimer(), - TimerWriterGet: metrics.NewTimer(), - TimerWriterIterator: metrics.NewTimer(), - TimerWriterSet: metrics.NewTimer(), - TimerWriterDelete: metrics.NewTimer(), - TimerIteratorSeekFirst: metrics.NewTimer(), - TimerIteratorSeek: metrics.NewTimer(), - TimerIteratorNext: metrics.NewTimer(), - TimerBatchMerge: metrics.NewTimer(), - TimerBatchExecute: metrics.NewTimer(), - - errors: list.New(), - } -} - -// The following structs are wrappers around "real" bleve kvstore -// implementations. - -type Store struct { - o store.KVStore - - TimerReaderGet metrics.Timer - TimerReaderIterator metrics.Timer - TimerWriterGet metrics.Timer - TimerWriterIterator metrics.Timer - TimerWriterSet metrics.Timer - TimerWriterDelete metrics.Timer - TimerIteratorSeekFirst metrics.Timer - TimerIteratorSeek metrics.Timer - TimerIteratorNext metrics.Timer - TimerBatchMerge metrics.Timer - TimerBatchExecute metrics.Timer - - m sync.Mutex // Protects the fields that follow. - errors *list.List // Capped list of StoreError's. -} - -type StoreError struct { - Time string - Op string - Err string - Key string -} - -type Reader struct { - s *Store - o store.KVReader -} - -type Writer struct { - s *Store - o store.KVWriter -} - -type Iterator struct { - s *Store - o store.KVIterator -} - -type Batch struct { - s *Store - o store.KVBatch -} - -func (s *Store) Close() error { - return s.o.Close() -} - -func (s *Store) Reader() (store.KVReader, error) { - o, err := s.o.Reader() - if err != nil { - s.AddError("Reader", err, nil) - return nil, err - } - return &Reader{s: s, o: o}, nil -} - -func (s *Store) Writer() (store.KVWriter, error) { - o, err := s.o.Writer() - if err != nil { - s.AddError("Writer", err, nil) - return nil, err - } - return &Writer{s: s, o: o}, nil -} - -func (s *Store) Actual() store.KVStore { - return s.o -} - -func (w *Reader) BytesSafeAfterClose() bool { - return w.o.BytesSafeAfterClose() -} - -func (w *Reader) Get(key []byte) (v []byte, err error) { - w.s.TimerReaderGet.Time(func() { - v, err = w.o.Get(key) - if err != nil { - w.s.AddError("Reader.Get", err, key) - } - }) - return -} - -func (w *Reader) Iterator(key []byte) (i store.KVIterator) { - w.s.TimerReaderIterator.Time(func() { - i = &Iterator{s: w.s, o: w.o.Iterator(key)} - }) - return -} - -func (w *Reader) Close() error { - err := w.o.Close() - if err != nil { - w.s.AddError("Reader.Close", err, nil) - } - return err -} - -func (w *Writer) BytesSafeAfterClose() bool { - return w.o.BytesSafeAfterClose() -} - -func (w *Writer) Get(key []byte) (v []byte, err error) { - w.s.TimerWriterGet.Time(func() { - v, err = w.o.Get(key) - if err != nil { - w.s.AddError("Writer.Get", err, key) - } - }) - return -} - -func (w *Writer) Iterator(key []byte) (i store.KVIterator) { - w.s.TimerWriterIterator.Time(func() { - i = &Iterator{s: w.s, o: w.o.Iterator(key)} - }) - return -} - -func (w *Writer) Close() error { - err := w.o.Close() - if err != nil { - w.s.AddError("Writer.Close", err, nil) - } - return err -} - -func (w *Writer) Set(key, val []byte) (err error) { - w.s.TimerWriterSet.Time(func() { - err = w.o.Set(key, val) - if err != nil { - w.s.AddError("Writer.Set", err, key) - } - }) - return -} - -func (w *Writer) Delete(key []byte) (err error) { - w.s.TimerWriterDelete.Time(func() { - err = w.o.Delete(key) - if err != nil { - w.s.AddError("Writer.Delete", err, key) - } - }) - return -} - -func (w *Writer) NewBatch() store.KVBatch { - return &Batch{s: w.s, o: w.o.NewBatch()} -} - -func (w *Iterator) SeekFirst() { - w.s.TimerIteratorSeekFirst.Time(func() { - w.o.SeekFirst() - }) -} - -func (w *Iterator) Seek(x []byte) { - w.s.TimerIteratorSeek.Time(func() { - w.o.Seek(x) - }) -} - -func (w *Iterator) Next() { - w.s.TimerIteratorNext.Time(func() { - w.o.Next() - }) -} - -func (w *Iterator) Current() ([]byte, []byte, bool) { - return w.o.Current() -} - -func (w *Iterator) Key() []byte { - return w.o.Key() -} - -func (w *Iterator) Value() []byte { - return w.o.Value() -} - -func (w *Iterator) Valid() bool { - return w.o.Valid() -} - -func (w *Iterator) Close() error { - err := w.o.Close() - if err != nil { - w.s.AddError("Iterator.Close", err, nil) - } - return err -} - -func (w *Batch) Set(key, val []byte) { - w.o.Set(key, val) -} - -func (w *Batch) Delete(key []byte) { - w.o.Delete(key) -} - -func (w *Batch) Merge(key []byte, oper store.AssociativeMerge) { - w.s.TimerBatchMerge.Time(func() { - w.o.Merge(key, oper) - }) -} - -func (w *Batch) Execute() (err error) { - w.s.TimerBatchExecute.Time(func() { - err = w.o.Execute() - if err != nil { - w.s.AddError("Batch.Execute", err, nil) - } - }) - return -} - -func (w *Batch) Close() error { - err := w.o.Close() - if err != nil { - w.s.AddError("Batch.Close", err, nil) - } - return err -} - -// -------------------------------------------------------- - -func (s *Store) AddError(op string, err error, key []byte) { - e := &StoreError{ - Time: time.Now().Format(time.RFC3339Nano), - Op: op, - Err: fmt.Sprintf("%v", err), - Key: string(key), - } - - s.m.Lock() - for s.errors.Len() >= MaxErrors { - s.errors.Remove(s.errors.Front()) - } - s.errors.PushBack(e) - s.m.Unlock() -} - -// -------------------------------------------------------- - -func (s *Store) WriteJSON(w io.Writer) { - w.Write([]byte(`{"TimerReaderGet":`)) - WriteTimerJSON(w, s.TimerReaderGet) - w.Write([]byte(`,"TimerReaderIterator":`)) - WriteTimerJSON(w, s.TimerReaderIterator) - w.Write([]byte(`,"TimerWriterGet":`)) - WriteTimerJSON(w, s.TimerWriterGet) - w.Write([]byte(`,"TimerWriterIterator":`)) - WriteTimerJSON(w, s.TimerWriterIterator) - w.Write([]byte(`,"TimerWriterSet":`)) - WriteTimerJSON(w, s.TimerWriterSet) - w.Write([]byte(`,"TimerWriterDelete":`)) - WriteTimerJSON(w, s.TimerWriterDelete) - w.Write([]byte(`,"TimerIteratorSeekFirst":`)) - WriteTimerJSON(w, s.TimerIteratorSeekFirst) - w.Write([]byte(`,"TimerIteratorSeek":`)) - WriteTimerJSON(w, s.TimerIteratorSeek) - w.Write([]byte(`,"TimerIteratorNext":`)) - WriteTimerJSON(w, s.TimerIteratorNext) - w.Write([]byte(`,"TimerBatchMerge":`)) - WriteTimerJSON(w, s.TimerBatchMerge) - w.Write([]byte(`,"TimerBatchExecute":`)) - WriteTimerJSON(w, s.TimerBatchExecute) - - w.Write([]byte(`,"Errors":[`)) - s.m.Lock() - e := s.errors.Front() - i := 0 - for e != nil { - se, ok := e.Value.(*StoreError) - if ok && se != nil { - if i > 0 { - w.Write([]byte(",")) - } - buf, err := json.Marshal(se) - if err == nil { - w.Write(buf) - } - } - e = e.Next() - i = i + 1 - } - s.m.Unlock() - w.Write([]byte(`]`)) - - w.Write([]byte(`}`)) -} - -func (s *Store) WriteCSVHeader(w io.Writer) { - WriteTimerCSVHeader(w, "TimerReaderGet") - WriteTimerCSVHeader(w, "TimerReaderIterator") - WriteTimerCSVHeader(w, "TimerWriterGet") - WriteTimerCSVHeader(w, "TimerWriterIterator") - WriteTimerCSVHeader(w, "TimerWriterSet") - WriteTimerCSVHeader(w, "TimerWriterDelete") - WriteTimerCSVHeader(w, "TimerIteratorSeekFirst") - WriteTimerCSVHeader(w, "TimerIteratorSeek") - WriteTimerCSVHeader(w, "TimerIteratorNext") - WriteTimerCSVHeader(w, "TimerBatchMerge") - WriteTimerCSVHeader(w, "TimerBatchExecute") -} - -func (s *Store) WriteCSV(w io.Writer) { - WriteTimerCSV(w, s.TimerReaderGet) - WriteTimerCSV(w, s.TimerReaderIterator) - WriteTimerCSV(w, s.TimerWriterGet) - WriteTimerCSV(w, s.TimerWriterIterator) - WriteTimerCSV(w, s.TimerWriterSet) - WriteTimerCSV(w, s.TimerWriterDelete) - WriteTimerCSV(w, s.TimerIteratorSeekFirst) - WriteTimerCSV(w, s.TimerIteratorSeek) - WriteTimerCSV(w, s.TimerIteratorNext) - WriteTimerCSV(w, s.TimerBatchMerge) - WriteTimerCSV(w, s.TimerBatchExecute) -} - -// -------------------------------------------------------- - -// NOTE: This is copy & pasted from cbft as otherwise there -// would be an import cycle. - -var timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999} - -func WriteTimerJSON(w io.Writer, timer metrics.Timer) { - t := timer.Snapshot() - p := t.Percentiles(timerPercentiles) - - fmt.Fprintf(w, `{"count":%9d,`, t.Count()) - fmt.Fprintf(w, `"min":%9d,`, t.Min()) - fmt.Fprintf(w, `"max":%9d,`, t.Max()) - fmt.Fprintf(w, `"mean":%12.2f,`, t.Mean()) - fmt.Fprintf(w, `"stddev":%12.2f,`, t.StdDev()) - fmt.Fprintf(w, `"percentiles":{`) - fmt.Fprintf(w, `"median":%12.2f,`, p[0]) - fmt.Fprintf(w, `"75%%":%12.2f,`, p[1]) - fmt.Fprintf(w, `"95%%":%12.2f,`, p[2]) - fmt.Fprintf(w, `"99%%":%12.2f,`, p[3]) - fmt.Fprintf(w, `"99.9%%":%12.2f},`, p[4]) - fmt.Fprintf(w, `"rates":{`) - fmt.Fprintf(w, `"1-min":%12.2f,`, t.Rate1()) - fmt.Fprintf(w, `"5-min":%12.2f,`, t.Rate5()) - fmt.Fprintf(w, `"15-min":%12.2f,`, t.Rate15()) - fmt.Fprintf(w, `"mean":%12.2f}}`, t.RateMean()) -} - -func WriteTimerCSVHeader(w io.Writer, prefix string) { - fmt.Fprintf(w, "%s-count,", prefix) - fmt.Fprintf(w, "%s-min,", prefix) - fmt.Fprintf(w, "%s-max,", prefix) - fmt.Fprintf(w, "%s-mean,", prefix) - fmt.Fprintf(w, "%s-stddev,", prefix) - fmt.Fprintf(w, "%s-percentile-50%%,", prefix) - fmt.Fprintf(w, "%s-percentile-75%%,", prefix) - fmt.Fprintf(w, "%s-percentile-95%%,", prefix) - fmt.Fprintf(w, "%s-percentile-99%%,", prefix) - fmt.Fprintf(w, "%s-percentile-99.9%%,", prefix) - fmt.Fprintf(w, "%s-rate-1-min,", prefix) - fmt.Fprintf(w, "%s-rate-5-min,", prefix) - fmt.Fprintf(w, "%s-rate-15-min,", prefix) - fmt.Fprintf(w, "%s-rate-mean", prefix) -} - -func WriteTimerCSV(w io.Writer, timer metrics.Timer) { - t := timer.Snapshot() - p := t.Percentiles(timerPercentiles) - - fmt.Fprintf(w, `%d,`, t.Count()) - fmt.Fprintf(w, `%d,`, t.Min()) - fmt.Fprintf(w, `%d,`, t.Max()) - fmt.Fprintf(w, `%f,`, t.Mean()) - fmt.Fprintf(w, `%f,`, t.StdDev()) - fmt.Fprintf(w, `%f,`, p[0]) - fmt.Fprintf(w, `%f,`, p[1]) - fmt.Fprintf(w, `%f,`, p[2]) - fmt.Fprintf(w, `%f,`, p[3]) - fmt.Fprintf(w, `%f,`, p[4]) - fmt.Fprintf(w, `%f,`, t.Rate1()) - fmt.Fprintf(w, `%f,`, t.Rate5()) - fmt.Fprintf(w, `%f,`, t.Rate15()) - fmt.Fprintf(w, `%f`, t.RateMean()) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/metrics/metrics_test.go b/vendor/github.com/blevesearch/bleve/index/store/metrics/metrics_test.go deleted file mode 100644 index 957aa9a6..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/metrics/metrics_test.go +++ /dev/null @@ -1,368 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the -// License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an "AS -// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language -// governing permissions and limitations under the License. - -// +build debug - -package metrics - -import ( - "bytes" - "encoding/json" - "fmt" - "reflect" - "testing" - - "github.com/blevesearch/bleve/index/store" - _ "github.com/blevesearch/bleve/index/store/gtreap" -) - -func TestMetricsStore(t *testing.T) { - s, err := StoreConstructor(map[string]interface{}{}) - if err == nil { - t.Errorf("expected err when bad config") - } - - s, err = StoreConstructor(map[string]interface{}{ - "kvStoreName_actual": "some-invalid-kvstore-name", - }) - if err == nil { - t.Errorf("expected err when unknown kvStoreName_actual") - } - - s, err = StoreConstructor(map[string]interface{}{ - "kvStoreName_actual": "gtreap", - }) - if err != nil { - t.Fatal(err) - } - - CommonTestKVStore(t, s) - - b := bytes.NewBuffer(nil) - s.(*Store).WriteJSON(b) - if b.Len() <= 0 { - t.Errorf("expected some output from WriteJSON") - } - var m map[string]interface{} - err = json.Unmarshal(b.Bytes(), &m) - if err != nil { - t.Errorf("expected WriteJSON to be unmarshallable") - } - if len(m) <= 0 { - t.Errorf("expected some entries") - } - - b = bytes.NewBuffer(nil) - s.(*Store).WriteCSVHeader(b) - if b.Len() <= 0 { - t.Errorf("expected some output from WriteCSVHeader") - } - - b = bytes.NewBuffer(nil) - s.(*Store).WriteCSV(b) - if b.Len() <= 0 { - t.Errorf("expected some output from WriteCSV") - } -} - -func TestReaderIsolation(t *testing.T) { - s, err := StoreConstructor(map[string]interface{}{ - "kvStoreName_actual": "gtreap", - }) - if err != nil { - t.Fatal(err) - } - - CommonTestReaderIsolation(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "b" { - t.Fatalf("expected key b, got %s", key) - } - if string(val) != "val-b" { - t.Fatalf("expected value val-b, got %s", val) - } - - it.Next() - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "c" { - t.Fatalf("expected key c, got %s", key) - } - if string(val) != "val-c" { - t.Fatalf("expected value val-c, got %s", val) - } - - it.Seek([]byte("i")) - key, val, valid = it.Current() - if !valid { - t.Fatalf("valid false, expected true") - } - if string(key) != "i" { - t.Fatalf("expected key i, got %s", key) - } - if string(val) != "val-i" { - t.Fatalf("expected value val-i, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } -} - -func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { - // insert a kv pair - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // create an isolated reader - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // verify that we see the value already inserted - val, err := reader.Get([]byte("a")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-a")) { - t.Errorf("expected val-a, got nil") - } - - // verify that an iterator sees it - count := 0 - it := reader.Iterator([]byte{0}) - defer func() { - err := it.Close() - if err != nil { - t.Fatal(err) - } - }() - for it.Valid() { - it.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } - - // add something after the reader was created - writer, err = s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("b"), []byte("val-b")) - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - // ensure that a newer reader sees it - newReader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := newReader.Close() - if err != nil { - t.Fatal(err) - } - }() - val, err = newReader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(val, []byte("val-b")) { - t.Errorf("expected val-b, got nil") - } - - // ensure that the director iterator sees it - count = 0 - it2 := newReader.Iterator([]byte{0}) - defer func() { - err := it2.Close() - if err != nil { - t.Fatal(err) - } - }() - for it2.Valid() { - it2.Next() - count++ - } - if count != 2 { - t.Errorf("expected iterator to see 2, saw %d", count) - } - - // but that the isolated reader does not - val, err = reader.Get([]byte("b")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %v", val) - } - - // and ensure that the iterator on the isolated reader also does not - count = 0 - it3 := reader.Iterator([]byte{0}) - defer func() { - err := it3.Close() - if err != nil { - t.Fatal(err) - } - }() - for it3.Valid() { - it3.Next() - count++ - } - if count != 1 { - t.Errorf("expected iterator to see 1, saw %d", count) - } -} - -func TestErrors(t *testing.T) { - s, err := StoreConstructor(map[string]interface{}{ - "kvStoreName_actual": "gtreap", - }) - if err != nil { - t.Fatal(err) - } - - x, ok := s.(*Store) - if !ok { - t.Errorf("expecting a Store") - } - - x.AddError("foo", fmt.Errorf("Foo"), []byte("fooKey")) - x.AddError("bar", fmt.Errorf("Bar"), nil) - x.AddError("baz", fmt.Errorf("Baz"), []byte("bazKey")) - - b := bytes.NewBuffer(nil) - x.WriteJSON(b) - - var m map[string]interface{} - err = json.Unmarshal(b.Bytes(), &m) - if err != nil { - t.Errorf("expected unmarshallable writeJSON, err: %v, b: %s", - err, b.Bytes()) - } - - errorsi, ok := m["Errors"] - if !ok || errorsi == nil { - t.Errorf("expected errorsi") - } - errors, ok := errorsi.([]interface{}) - if !ok || errors == nil { - t.Errorf("expected errorsi is array") - } - if len(errors) != 3 { - t.Errorf("expected errors len 3") - } - - e := errors[0].(map[string]interface{}) - if e["Op"].(string) != "foo" || - e["Err"].(string) != "Foo" || - len(e["Time"].(string)) < 10 || - e["Key"].(string) != "fooKey" { - t.Errorf("expected foo, %#v", e) - } - e = errors[1].(map[string]interface{}) - if e["Op"].(string) != "bar" || - e["Err"].(string) != "Bar" || - len(e["Time"].(string)) < 10 || - e["Key"].(string) != "" { - t.Errorf("expected bar, %#v", e) - } - e = errors[2].(map[string]interface{}) - if e["Op"].(string) != "baz" || - e["Err"].(string) != "Baz" || - len(e["Time"].(string)) < 10 || - e["Key"].(string) != "bazKey" { - t.Errorf("expected baz, %#v", e) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/multiget.go b/vendor/github.com/blevesearch/bleve/index/store/multiget.go new file mode 100644 index 00000000..635bcd41 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/store/multiget.go @@ -0,0 +1,33 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package store + +// MultiGet is a helper function to retrieve mutiple keys from a +// KVReader, and might be used by KVStore implementations that don't +// have a native multi-get facility. +func MultiGet(kvreader KVReader, keys [][]byte) ([][]byte, error) { + vals := make([][]byte, 0, len(keys)) + + for i, key := range keys { + val, err := kvreader.Get(key) + if err != nil { + return nil, err + } + + vals[i] = val + } + + return vals, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/store/null/null.go b/vendor/github.com/blevesearch/bleve/index/store/null/null.go deleted file mode 100644 index 025c8a91..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/null/null.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package null - -import ( - "github.com/blevesearch/bleve/index/store" -) - -type Store struct{} - -func New() (*Store, error) { - rv := Store{} - return &rv, nil -} - -func (i *Store) Open() error { - return nil -} - -func (i *Store) SetMergeOperator(mo store.MergeOperator) { - -} - -func (i *Store) Close() error { - return nil -} - -func (i *Store) iterator(key []byte) store.KVIterator { - rv := newIterator(i) - return rv -} - -func (i *Store) Reader() (store.KVReader, error) { - return newReader(i) -} - -func (i *Store) Writer() (store.KVWriter, error) { - return newWriter(i) -} - -func (i *Store) newBatch() store.KVBatch { - return newBatch(i) -} - -type Reader struct { - store *Store -} - -func newReader(store *Store) (*Reader, error) { - return &Reader{ - store: store, - }, nil -} - -func (r *Reader) BytesSafeAfterClose() bool { - return true -} - -func (r *Reader) Get(key []byte) ([]byte, error) { - return nil, nil -} - -func (r *Reader) Iterator(key []byte) store.KVIterator { - return r.store.iterator(key) -} - -func (r *Reader) Close() error { - return nil -} - -type Iterator struct{} - -func newIterator(store *Store) *Iterator { - return &Iterator{} -} - -func (i *Iterator) SeekFirst() {} - -func (i *Iterator) Seek(k []byte) {} - -func (i *Iterator) Next() {} - -func (i *Iterator) Current() ([]byte, []byte, bool) { - return nil, nil, false -} - -func (i *Iterator) Key() []byte { - return nil -} - -func (i *Iterator) Value() []byte { - return nil -} - -func (i *Iterator) Valid() bool { - return false -} - -func (i *Iterator) Close() error { - return nil -} - -type Batch struct{} - -func newBatch(s *Store) *Batch { - rv := Batch{} - return &rv -} - -func (i *Batch) Set(key, val []byte) { -} - -func (i *Batch) Delete(key []byte) { -} - -func (i *Batch) Merge(key, val []byte) { -} - -func (i *Batch) Execute() error { - return nil -} - -func (i *Batch) Close() error { - return nil -} - -type Writer struct { - store *Store -} - -func newWriter(store *Store) (*Writer, error) { - return &Writer{ - store: store, - }, nil -} - -func (w *Writer) BytesSafeAfterClose() bool { - return true -} - -func (w *Writer) Set(key, val []byte) error { - return nil -} - -func (w *Writer) Delete(key []byte) error { - return nil -} - -func (w *Writer) NewBatch() store.KVBatch { - return newBatch(w.store) -} - -func (w *Writer) Close() error { - return nil -} - -// these two methods can safely read using the regular -// methods without a read transaction, because we know -// that no one else is writing but us -func (w *Writer) Get(key []byte) ([]byte, error) { - return nil, nil -} - -func (w *Writer) Iterator(key []byte) store.KVIterator { - return w.store.iterator(key) -} diff --git a/vendor/github.com/blevesearch/bleve/index/store/null/null_test.go b/vendor/github.com/blevesearch/bleve/index/store/null/null_test.go deleted file mode 100644 index 720d482a..00000000 --- a/vendor/github.com/blevesearch/bleve/index/store/null/null_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package null - -import ( - "testing" - - "github.com/blevesearch/bleve/index/store" -) - -func TestStore(t *testing.T) { - s, err := New() - if err != nil { - t.Fatal(err) - } - - CommonTestKVStore(t, s) -} - -func CommonTestKVStore(t *testing.T, s store.KVStore) { - - writer, err := s.Writer() - if err != nil { - t.Error(err) - } - err = writer.Set([]byte("a"), []byte("val-a")) - if err != nil { - t.Fatal(err) - } - err = writer.Set([]byte("z"), []byte("val-z")) - if err != nil { - t.Fatal(err) - } - err = writer.Delete([]byte("z")) - if err != nil { - t.Fatal(err) - } - - batch := writer.NewBatch() - batch.Set([]byte("b"), []byte("val-b")) - batch.Set([]byte("c"), []byte("val-c")) - batch.Set([]byte("d"), []byte("val-d")) - batch.Set([]byte("e"), []byte("val-e")) - batch.Set([]byte("f"), []byte("val-f")) - batch.Set([]byte("g"), []byte("val-g")) - batch.Set([]byte("h"), []byte("val-h")) - batch.Set([]byte("i"), []byte("val-i")) - batch.Set([]byte("j"), []byte("val-j")) - - err = batch.Execute() - if err != nil { - t.Fatal(err) - } - err = writer.Close() - if err != nil { - t.Fatal(err) - } - - reader, err := s.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - it := reader.Iterator([]byte("b")) - key, val, valid := it.Current() - if valid { - t.Fatalf("valid true, expected false") - } - if key != nil { - t.Fatalf("expected key nil, got %s", key) - } - if val != nil { - t.Fatalf("expected value nil, got %s", val) - } - - err = it.Close() - if err != nil { - t.Fatal(err) - } - - err = s.Close() - if err != nil { - t.Fatal(err) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/analysis_pool.go b/vendor/github.com/blevesearch/bleve/index/upside_down/analysis_pool.go deleted file mode 100644 index 081136b4..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/analysis_pool.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "github.com/blevesearch/bleve/document" -) - -type AnalysisResult struct { - docID string - rows []UpsideDownCouchRow -} - -type AnalysisWork struct { - udc *UpsideDownCouch - d *document.Document - rc chan *AnalysisResult -} - -type AnalysisQueue struct { - queue chan *AnalysisWork - done chan struct{} -} - -func (q *AnalysisQueue) Queue(work *AnalysisWork) { - q.queue <- work -} - -func (q *AnalysisQueue) Close() { - close(q.done) -} - -func NewAnalysisQueue(numWorkers int) *AnalysisQueue { - rv := AnalysisQueue{ - queue: make(chan *AnalysisWork), - done: make(chan struct{}), - } - for i := 0; i < numWorkers; i++ { - go AnalysisWorker(rv) - } - return &rv -} - -func AnalysisWorker(q AnalysisQueue) { - // read work off the queue - for { - select { - case <-q.done: - return - case w := <-q.queue: - - rv := &AnalysisResult{ - docID: w.d.ID, - rows: make([]UpsideDownCouchRow, 0, 100), - } - - // track our back index entries - backIndexTermEntries := make([]*BackIndexTermEntry, 0) - backIndexStoredEntries := make([]*BackIndexStoreEntry, 0) - - for _, field := range w.d.Fields { - fieldIndex, newFieldRow := w.udc.fieldIndexCache.FieldIndex(field.Name()) - if newFieldRow != nil { - rv.rows = append(rv.rows, newFieldRow) - } - - if field.Options().IsIndexed() { - - fieldLength, tokenFreqs := field.Analyze() - - // see if any of the composite fields need this - for _, compositeField := range w.d.CompositeFields { - compositeField.Compose(field.Name(), fieldLength, tokenFreqs) - } - - // encode this field - indexRows, indexBackIndexTermEntries := w.udc.indexField(w.d.ID, field, fieldIndex, fieldLength, tokenFreqs) - rv.rows = append(rv.rows, indexRows...) - backIndexTermEntries = append(backIndexTermEntries, indexBackIndexTermEntries...) - } - - if field.Options().IsStored() { - storeRows, indexBackIndexStoreEntries := w.udc.storeField(w.d.ID, field, fieldIndex) - rv.rows = append(rv.rows, storeRows...) - backIndexStoredEntries = append(backIndexStoredEntries, indexBackIndexStoreEntries...) - } - - } - - // now index the composite fields - for _, compositeField := range w.d.CompositeFields { - fieldIndex, newFieldRow := w.udc.fieldIndexCache.FieldIndex(compositeField.Name()) - if newFieldRow != nil { - rv.rows = append(rv.rows, newFieldRow) - } - if compositeField.Options().IsIndexed() { - fieldLength, tokenFreqs := compositeField.Analyze() - // encode this field - indexRows, indexBackIndexTermEntries := w.udc.indexField(w.d.ID, compositeField, fieldIndex, fieldLength, tokenFreqs) - rv.rows = append(rv.rows, indexRows...) - backIndexTermEntries = append(backIndexTermEntries, indexBackIndexTermEntries...) - } - } - - // build the back index row - backIndexRow := NewBackIndexRow(w.d.ID, backIndexTermEntries, backIndexStoredEntries) - rv.rows = append(rv.rows, backIndexRow) - - w.rc <- rv - - } - - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_boltdb_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_boltdb_test.go deleted file mode 100644 index 549f8d7b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_boltdb_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "os" - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/boltdb" -) - -func CreateBoltDB() (store.KVStore, error) { - s := boltdb.New("test", "bleve") - return s, nil -} - -func DestroyBoltDB() error { - return os.RemoveAll("test") -} - -func BenchmarkBoltDBIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 1) -} - -func BenchmarkBoltDBIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 2) -} - -func BenchmarkBoltDBIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 4) -} - -// batches - -func BenchmarkBoltDBIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 10) -} - -func BenchmarkBoltDBIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 10) -} - -func BenchmarkBoltDBIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 10) -} - -func BenchmarkBoltDBIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 100) -} - -func BenchmarkBoltDBIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 100) -} - -func BenchmarkBoltDBIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 100) -} - -func BenchmarkBoltBIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 1000) -} - -func BenchmarkBoltBIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 1000) -} - -func BenchmarkBoltBIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_common_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_common_test.go deleted file mode 100644 index a1946a03..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_common_test.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "strconv" - "testing" - - _ "github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/registry" -) - -var benchmarkDocBodies = []string{ - "A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.", - "A boiler explosion is a catastrophic failure of a boiler. As seen today, boiler explosions are of two kinds. One kind is a failure of the pressure parts of the steam and water sides. There can be many different causes, such as failure of the safety valve, corrosion of critical parts of the boiler, or low water level. Corrosion along the edges of lap joints was a common cause of early boiler explosions.", - "A boiler is a closed vessel in which water or other fluid is heated. The fluid does not necessarily boil. (In North America the term \"furnace\" is normally used if the purpose is not actually to boil the fluid.) The heated or vaporized fluid exits the boiler for use in various processes or heating applications,[1][2] including central heating, boiler-based power generation, cooking, and sanitation.", - "A pressure vessel is a closed container designed to hold gases or liquids at a pressure substantially different from the ambient pressure.", - "Pressure (symbol: p or P) is the ratio of force to the area over which that force is distributed.", - "Liquid is one of the four fundamental states of matter (the others being solid, gas, and plasma), and is the only state with a definite volume but no fixed shape.", - "The boiling point of a substance is the temperature at which the vapor pressure of the liquid equals the pressure surrounding the liquid[1][2] and the liquid changes into a vapor.", - "Vapor pressure or equilibrium vapor pressure is defined as the pressure exerted by a vapor in thermodynamic equilibrium with its condensed phases (solid or liquid) at a given temperature in a closed system.", - "Industrial gases are a group of gases that are specifically manufactured for use in a wide range of industries, which include oil and gas, petrochemicals, chemicals, power, mining, steelmaking, metals, environmental protection, medicine, pharmaceuticals, biotechnology, food, water, fertilizers, nuclear power, electronics and aerospace.", - "The expansion ratio of a liquefied and cryogenic substance is the volume of a given amount of that substance in liquid form compared to the volume of the same amount of substance in gaseous form, at room temperature and normal atmospheric pressure.", -} - -type KVStoreCreate func() (store.KVStore, error) -type KVStoreDestroy func() error - -func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers int) { - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed("standard") - if err != nil { - b.Fatal(err) - } - - indexDocument := document.NewDocument(""). - AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer)) - - b.ResetTimer() - b.StopTimer() - for i := 0; i < b.N; i++ { - s, err := create() - if err != nil { - b.Fatal(err) - } - analysisQueue := NewAnalysisQueue(analysisWorkers) - idx := NewUpsideDownCouch(s, analysisQueue) - - err = idx.Open() - if err != nil { - b.Fatal(err) - } - indexDocument.ID = strconv.Itoa(i) - // just time the indexing portion - b.StartTimer() - err = idx.Update(indexDocument) - if err != nil { - b.Fatal(err) - } - b.StopTimer() - err = idx.Close() - if err != nil { - b.Fatal(err) - } - err = destroy() - if err != nil { - b.Fatal(err) - } - analysisQueue.Close() - } -} - -func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) { - - cache := registry.NewCache() - analyzer, err := cache.AnalyzerNamed("standard") - if err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.StopTimer() - for i := 0; i < b.N; i++ { - - s, err := create() - if err != nil { - b.Fatal(err) - } - analysisQueue := NewAnalysisQueue(analysisWorkers) - idx := NewUpsideDownCouch(s, analysisQueue) - - err = idx.Open() - if err != nil { - b.Fatal(err) - } - - b.StartTimer() - batch := index.NewBatch() - for j := 0; j < 1000; j++ { - if j%batchSize == 0 { - if len(batch.IndexOps) > 0 { - err := idx.Batch(batch) - if err != nil { - b.Fatal(err) - } - } - batch = index.NewBatch() - } - indexDocument := document.NewDocument(""). - AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer)) - indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j) - batch.Update(indexDocument) - } - // close last batch - if len(batch.IndexOps) > 0 { - err := idx.Batch(batch) - if err != nil { - b.Fatal(err) - } - } - b.StopTimer() - err = idx.Close() - if err != nil { - b.Fatal(err) - } - err = destroy() - if err != nil { - b.Fatal(err) - } - analysisQueue.Close() - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_cznicb_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_cznicb_test.go deleted file mode 100644 index 82eed6ff..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_cznicb_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/cznicb" -) - -func CreateCznicB() (store.KVStore, error) { - return cznicb.StoreConstructor(nil) -} - -func DestroyCznicB() error { - return nil -} - -func BenchmarkCznicBIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 1) -} - -func BenchmarkCznicBIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 2) -} - -func BenchmarkCznicBIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 4) -} - -// batches - -func BenchmarkCznicBIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 10) -} - -func BenchmarkCznicBIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 10) -} - -func BenchmarkCznicBIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 10) -} - -func BenchmarkCznicBIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 100) -} - -func BenchmarkCznicBIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 100) -} - -func BenchmarkCznicBIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 100) -} - -func BenchmarkCznicBIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 1000) -} - -func BenchmarkCznicBIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 1000) -} - -func BenchmarkCznicBIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_forestdb_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_forestdb_test.go deleted file mode 100644 index 31b33165..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_forestdb_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build forestdb - -package upside_down - -import ( - "os" - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/forestdb" -) - -func CreateForestDB() (store.KVStore, error) { - err := os.MkdirAll("testdir", 0700) - if err != nil { - return nil, err - } - s, err := forestdb.New("testdir/test", true, nil) - if err != nil { - return nil, err - } - return s, nil -} - -func DestroyForestDB() error { - return os.RemoveAll("testdir") -} - -func BenchmarkForestDBIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 1) -} - -func BenchmarkForestDBIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 2) -} - -func BenchmarkForestDBIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 4) -} - -// batches - -func BenchmarkForestDBIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 10) -} - -func BenchmarkForestDBIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 10) -} - -func BenchmarkForestDBIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 10) -} - -func BenchmarkForestDBIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 100) -} - -func BenchmarkForestDBIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 100) -} - -func BenchmarkForestDBIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 100) -} - -func BenchmarkForestDBIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 1000) -} - -func BenchmarkForestDBIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 1000) -} - -func BenchmarkForestDBIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_goleveldb_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_goleveldb_test.go deleted file mode 100644 index ba003367..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_goleveldb_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "os" - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/goleveldb" -) - -var goLevelDBTestOptions = map[string]interface{}{ - "create_if_missing": true, -} - -func CreateGoLevelDB() (store.KVStore, error) { - return goleveldb.New("test", goLevelDBTestOptions) -} - -func DestroyGoLevelDB() error { - return os.RemoveAll("test") -} - -func BenchmarkGoLevelDBIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 1) -} - -func BenchmarkGoLevelDBIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 2) -} - -func BenchmarkGoLevelDBIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 4) -} - -// batches - -func BenchmarkGoLevelDBIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 10) -} - -func BenchmarkGoLevelDBIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 10) -} - -func BenchmarkGoLevelDBIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 10) -} - -func BenchmarkGoLevelDBIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 100) -} - -func BenchmarkGoLevelDBIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 100) -} - -func BenchmarkGoLevelDBIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 100) -} - -func BenchmarkGoLevelDBIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 1000) -} - -func BenchmarkGoLevelDBIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 1000) -} - -func BenchmarkGoLevelDBIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gorocksdb_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gorocksdb_test.go deleted file mode 100644 index 4c1693f8..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gorocksdb_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build rocksdb - -package upside_down - -import ( - "os" - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/gorocksdb" -) - -var rocksdbTestOptions = map[string]interface{}{ - "create_if_missing": true, -} - -func CreateGoRocksDB() (store.KVStore, error) { - return rocksdb.New("test", rocksdbTestOptions) -} - -func DestroyGoRocksDB() error { - return os.RemoveAll("test") -} - -func BenchmarkRocksDBIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 1) -} - -func BenchmarkRocksDBIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 2) -} - -func BenchmarkRocksDBIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 4) -} - -// batches - -func BenchmarkRocksDBIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 10) -} - -func BenchmarkRocksDBIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 10) -} - -func BenchmarkRocksDBIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 10) -} - -func BenchmarkRocksDBIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 100) -} - -func BenchmarkRocksDBIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 100) -} - -func BenchmarkRocksDBIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 100) -} - -func BenchmarkRocksDBIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 1000) -} - -func BenchmarkRocksDBIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 1000) -} - -func BenchmarkRocksDBIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gtreap_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gtreap_test.go deleted file mode 100644 index 673d830f..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gtreap_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/gtreap" -) - -func CreateGTreap() (store.KVStore, error) { - return gtreap.StoreConstructor(nil) -} - -func DestroyGTreap() error { - return nil -} - -func BenchmarkGTreapIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 1) -} - -func BenchmarkGTreapIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 2) -} - -func BenchmarkGTreapIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 4) -} - -// batches - -func BenchmarkGTreapIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 10) -} - -func BenchmarkGTreapIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 10) -} - -func BenchmarkGTreapIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 10) -} - -func BenchmarkGTreapIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 100) -} - -func BenchmarkGTreapIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 100) -} - -func BenchmarkGTreapIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 100) -} - -func BenchmarkGTreapIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 1000) -} - -func BenchmarkGTreapIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 1000) -} - -func BenchmarkGTreapIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_inmem_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_inmem_test.go deleted file mode 100644 index ee06bd5b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_inmem_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/inmem" -) - -func CreateInMem() (store.KVStore, error) { - return inmem.New() -} - -func DestroyInMem() error { - return nil -} - -func BenchmarkInMemIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 1) -} - -func BenchmarkInMemIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 2) -} - -func BenchmarkInMemIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 4) -} - -// batches - -func BenchmarkInMemIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 10) -} - -func BenchmarkInMemIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 10) -} - -func BenchmarkInMemIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 10) -} - -func BenchmarkInMemIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 100) -} - -func BenchmarkInMemIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 100) -} - -func BenchmarkInMemIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 100) -} - -func BenchmarkInMemIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 1000) -} - -func BenchmarkInMemIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 1000) -} - -func BenchmarkInMemIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_leveldb_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_leveldb_test.go deleted file mode 100644 index 4e6a70ed..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_leveldb_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build leveldb full - -package upside_down - -import ( - "os" - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/leveldb" -) - -var leveldbTestOptions = map[string]interface{}{ - "create_if_missing": true, -} - -func CreateLevelDB() (store.KVStore, error) { - return leveldb.New("test", leveldbTestOptions) -} - -func DestroyLevelDB() error { - return os.RemoveAll("test") -} - -func BenchmarkLevelDBIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 1) -} - -func BenchmarkLevelDBIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 2) -} - -func BenchmarkLevelDBIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 4) -} - -// batches - -func BenchmarkLevelDBIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 10) -} - -func BenchmarkLevelDBIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 10) -} - -func BenchmarkLevelDBIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 10) -} - -func BenchmarkLevelDBIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 100) -} - -func BenchmarkLevelDBIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 100) -} - -func BenchmarkLevelDBIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 100) -} - -func BenchmarkLevelDBIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 1000) -} - -func BenchmarkLevelDBIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 1000) -} - -func BenchmarkLevelDBIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_null_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_null_test.go deleted file mode 100644 index a02813a3..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_null_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "testing" - - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/null" -) - -func CreateNull() (store.KVStore, error) { - return null.New() -} - -func DestroyNull() error { - return nil -} - -func BenchmarkNullIndexing1Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateNull, DestroyNull, 1) -} - -func BenchmarkNullIndexing2Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateNull, DestroyNull, 2) -} - -func BenchmarkNullIndexing4Workers(b *testing.B) { - CommonBenchmarkIndex(b, CreateNull, DestroyNull, 4) -} - -// batches - -func BenchmarkNullIndexing1Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 10) -} - -func BenchmarkNullIndexing2Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 10) -} - -func BenchmarkNullIndexing4Workers10Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 10) -} - -func BenchmarkNullIndexing1Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 100) -} - -func BenchmarkNullIndexing2Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 100) -} - -func BenchmarkNullIndexing4Workers100Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 100) -} - -func BenchmarkNullIndexing1Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 1000) -} - -func BenchmarkNullIndexing2Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 1000) -} - -func BenchmarkNullIndexing4Workers1000Batch(b *testing.B) { - CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 1000) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/dump.go b/vendor/github.com/blevesearch/bleve/index/upside_down/dump.go deleted file mode 100644 index 05333918..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/dump.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "bytes" - "sort" - - "github.com/blevesearch/bleve/index/store" -) - -// the functions in this file are only intended to be used by -// the bleve_dump utility and the debug http handlers -// if your application relies on them, you're doing something wrong -// they may change or be removed at any time - -func (udc *UpsideDownCouch) dumpPrefix(kvreader store.KVReader, rv chan interface{}, prefix []byte) { - start := prefix - if start == nil { - start = []byte{0} - } - it := kvreader.Iterator(start) - defer func() { - cerr := it.Close() - if cerr != nil { - rv <- cerr - } - }() - key, val, valid := it.Current() - for valid { - - if prefix != nil && !bytes.HasPrefix(key, prefix) { - break - } - - row, err := ParseFromKeyValue(key, val) - if err != nil { - rv <- err - return - } - rv <- row - - it.Next() - key, val, valid = it.Current() - } -} - -func (udc *UpsideDownCouch) DumpAll() chan interface{} { - rv := make(chan interface{}) - go func() { - defer close(rv) - - // start an isolated reader for use during the dump - kvreader, err := udc.store.Reader() - if err != nil { - rv <- err - return - } - defer func() { - cerr := kvreader.Close() - if cerr != nil { - rv <- cerr - } - }() - - udc.dumpPrefix(kvreader, rv, nil) - }() - return rv -} - -func (udc *UpsideDownCouch) DumpFields() chan interface{} { - rv := make(chan interface{}) - go func() { - defer close(rv) - - // start an isolated reader for use during the dump - kvreader, err := udc.store.Reader() - if err != nil { - rv <- err - return - } - defer func() { - cerr := kvreader.Close() - if cerr != nil { - rv <- cerr - } - }() - - udc.dumpPrefix(kvreader, rv, []byte{'f'}) - }() - return rv -} - -type keyset [][]byte - -func (k keyset) Len() int { return len(k) } -func (k keyset) Swap(i, j int) { k[i], k[j] = k[j], k[i] } -func (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 } - -// DumpDoc returns all rows in the index related to this doc id -func (udc *UpsideDownCouch) DumpDoc(id string) chan interface{} { - rv := make(chan interface{}) - - go func() { - defer close(rv) - - // start an isolated reader for use during the dump - kvreader, err := udc.store.Reader() - if err != nil { - rv <- err - return - } - defer func() { - cerr := kvreader.Close() - if cerr != nil { - rv <- cerr - } - }() - - back, err := udc.backIndexRowForDoc(kvreader, id) - if err != nil { - rv <- err - return - } - - // no such doc - if back == nil { - return - } - // build sorted list of term keys - keys := make(keyset, 0) - for _, entry := range back.termEntries { - tfr := NewTermFrequencyRow([]byte(*entry.Term), uint16(*entry.Field), id, 0, 0) - key := tfr.Key() - keys = append(keys, key) - } - sort.Sort(keys) - - // first add all the stored rows - storedRowPrefix := NewStoredRow(id, 0, []uint64{}, 'x', []byte{}).ScanPrefixForDoc() - udc.dumpPrefix(kvreader, rv, storedRowPrefix) - - // now walk term keys in order and add them as well - if len(keys) > 0 { - it := kvreader.Iterator(keys[0]) - defer func() { - cerr := it.Close() - if cerr != nil { - rv <- cerr - } - }() - - for _, key := range keys { - it.Seek(key) - rkey, rval, valid := it.Current() - if !valid { - break - } - row, err := ParseFromKeyValue(rkey, rval) - if err != nil { - rv <- err - return - } - rv <- row - } - } - }() - - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/dump_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/dump_test.go deleted file mode 100644 index ff77280b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/dump_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "github.com/blevesearch/bleve/index/store/boltdb" - "os" - "testing" - "time" - - "github.com/blevesearch/bleve/document" -) - -func TestDump(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s := boltdb.New("test", "bleve") - s.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(s, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - docCount, err := idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) - doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) - dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) - if err != nil { - t.Error(err) - } - doc.AddField(dateField) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - doc = document.NewDocument("2") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.IndexField|document.StoreField)) - doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) - dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) - if err != nil { - t.Error(err) - } - doc.AddField(dateField) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - fieldsCount := 0 - fieldsRows := idx.DumpFields() - for _ = range fieldsRows { - fieldsCount++ - } - if fieldsCount != 3 { - t.Errorf("expected 3 fields, got %d", fieldsCount) - } - - // 1 text term - // 16 numeric terms - // 16 date terms - // 3 stored fields - expectedDocRowCount := int(1 + (2 * (64 / document.DefaultPrecisionStep)) + 3) - docRowCount := 0 - docRows := idx.DumpDoc("1") - for _ = range docRows { - docRowCount++ - } - if docRowCount != expectedDocRowCount { - t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) - } - - docRowCount = 0 - docRows = idx.DumpDoc("2") - for _ = range docRows { - docRowCount++ - } - if docRowCount != expectedDocRowCount { - t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount) - } - - // 1 version - // fieldsCount field rows - // 2 docs * expectedDocRowCount - // 2 back index rows - // 2 text term row count (2 different text terms) - // 16 numeric term row counts (shared for both docs, same numeric value) - // 16 date term row counts (shared for both docs, same date value) - expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DefaultPrecisionStep)))) - allRowCount := 0 - allRows := idx.DumpAll() - for _ = range allRows { - allRowCount++ - } - if allRowCount != expectedAllRowCount { - t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/field_dict.go b/vendor/github.com/blevesearch/bleve/index/upside_down/field_dict.go deleted file mode 100644 index f5feb120..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/field_dict.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "bytes" - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" -) - -type UpsideDownCouchFieldDict struct { - indexReader *IndexReader - iterator store.KVIterator - endKey []byte - field uint16 -} - -func newUpsideDownCouchFieldDict(indexReader *IndexReader, field uint16, startTerm, endTerm []byte) (*UpsideDownCouchFieldDict, error) { - - startKey := NewDictionaryRow(startTerm, field, 0).Key() - if endTerm == nil { - endTerm = []byte{ByteSeparator} - } - endKey := NewDictionaryRow(endTerm, field, 0).Key() - - it := indexReader.kvreader.Iterator(startKey) - - return &UpsideDownCouchFieldDict{ - indexReader: indexReader, - iterator: it, - field: field, - endKey: endKey, - }, nil - -} - -func (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) { - key, val, valid := r.iterator.Current() - if !valid { - return nil, nil - } - - // past end term - if bytes.Compare(key, r.endKey) > 0 { - return nil, nil - } - - currRow, err := NewDictionaryRowKV(key, val) - if err != nil { - return nil, fmt.Errorf("unexpected error parsing dictionary row kv: %v", err) - } - rv := index.DictEntry{ - Term: string(currRow.term), - Count: currRow.count, - } - // advance the iterator to the next term - r.iterator.Next() - return &rv, nil - -} - -func (r *UpsideDownCouchFieldDict) Close() error { - return r.iterator.Close() -} - -func incrementBytes(in []byte) []byte { - rv := make([]byte, len(in)) - copy(rv, in) - for i := len(rv) - 1; i >= 0; i-- { - rv[i] = rv[i] + 1 - if rv[i] != 0 { - // didn't overflow, so stop - break - } - } - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/field_dict_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/field_dict_test.go deleted file mode 100644 index d5af8ca0..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/field_dict_test.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "os" - "reflect" - "testing" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index/store/boltdb" -) - -func TestIndexFieldDict(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s := boltdb.New("test", "bleve") - s.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(s, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - doc = document.NewDocument("2") - doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer)) - doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer)) - doc.AddField(document.NewTextFieldCustom("prefix", []uint64{}, []byte("bob cat cats catting dog doggy zoo"), document.IndexField|document.IncludeTermVectors, testAnalyzer)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - dict, err := indexReader.FieldDict("name") - if err != nil { - t.Errorf("error creating reader: %v", err) - } - defer func() { - err := dict.Close() - if err != nil { - t.Fatal(err) - } - }() - - termCount := 0 - curr, err := dict.Next() - for err == nil && curr != nil { - termCount++ - if curr.Term != "test" { - t.Errorf("expected term to be 'test', got '%s'", curr.Term) - } - curr, err = dict.Next() - } - if termCount != 1 { - t.Errorf("expected 1 term for this field, got %d", termCount) - } - - dict2, err := indexReader.FieldDict("desc") - if err != nil { - t.Errorf("error creating reader: %v", err) - } - defer func() { - err := dict2.Close() - if err != nil { - t.Fatal(err) - } - }() - - termCount = 0 - terms := make([]string, 0) - curr, err = dict2.Next() - for err == nil && curr != nil { - termCount++ - terms = append(terms, curr.Term) - curr, err = dict2.Next() - } - if termCount != 3 { - t.Errorf("expected 3 term for this field, got %d", termCount) - } - expectedTerms := []string{"eat", "more", "rice"} - if !reflect.DeepEqual(expectedTerms, terms) { - t.Errorf("expected %#v, got %#v", expectedTerms, terms) - } - - // test start and end range - dict3, err := indexReader.FieldDictRange("desc", []byte("fun"), []byte("nice")) - if err != nil { - t.Errorf("error creating reader: %v", err) - } - defer func() { - err := dict3.Close() - if err != nil { - t.Fatal(err) - } - }() - - termCount = 0 - terms = make([]string, 0) - curr, err = dict3.Next() - for err == nil && curr != nil { - termCount++ - terms = append(terms, curr.Term) - curr, err = dict3.Next() - } - if termCount != 1 { - t.Errorf("expected 1 term for this field, got %d", termCount) - } - expectedTerms = []string{"more"} - if !reflect.DeepEqual(expectedTerms, terms) { - t.Errorf("expected %#v, got %#v", expectedTerms, terms) - } - - // test use case for prefix - dict4, err := indexReader.FieldDictPrefix("prefix", []byte("cat")) - if err != nil { - t.Errorf("error creating reader: %v", err) - } - defer func() { - err := dict4.Close() - if err != nil { - t.Fatal(err) - } - }() - - termCount = 0 - terms = make([]string, 0) - curr, err = dict4.Next() - for err == nil && curr != nil { - termCount++ - terms = append(terms, curr.Term) - curr, err = dict4.Next() - } - if termCount != 3 { - t.Errorf("expected 3 term for this field, got %d", termCount) - } - expectedTerms = []string{"cat", "cats", "catting"} - if !reflect.DeepEqual(expectedTerms, terms) { - t.Errorf("expected %#v, got %#v", expectedTerms, terms) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/field_index_cache.go b/vendor/github.com/blevesearch/bleve/index/upside_down/field_index_cache.go deleted file mode 100644 index 8d0b23a7..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/field_index_cache.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "sync" -) - -type FieldIndexCache struct { - fieldIndexes map[string]uint16 - lastFieldIndex int - mutex sync.RWMutex -} - -func NewFieldIndexCache() *FieldIndexCache { - return &FieldIndexCache{ - fieldIndexes: make(map[string]uint16), - } -} - -func (f *FieldIndexCache) AddExisting(field string, index uint16) { - f.mutex.Lock() - defer f.mutex.Unlock() - f.fieldIndexes[field] = index - if int(index) > f.lastFieldIndex { - f.lastFieldIndex = int(index) - } -} - -func (f *FieldIndexCache) FieldExists(field string) (uint16, bool) { - f.mutex.RLock() - defer f.mutex.RUnlock() - if index, ok := f.fieldIndexes[field]; ok { - return index, true - } - return 0, false -} - -func (f *FieldIndexCache) FieldIndex(field string) (uint16, *FieldRow) { - f.mutex.Lock() - defer f.mutex.Unlock() - index, exists := f.fieldIndexes[field] - if exists { - return index, nil - } - // assign next field id - index = uint16(f.lastFieldIndex + 1) - f.fieldIndexes[field] = index - f.lastFieldIndex = int(index) - return index, NewFieldRow(uint16(index), field) -} - -func (f *FieldIndexCache) FieldName(index uint16) string { - f.mutex.RLock() - defer f.mutex.RUnlock() - for fieldName, fieldIndex := range f.fieldIndexes { - if index == fieldIndex { - return fieldName - } - } - return "" -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/index_reader.go b/vendor/github.com/blevesearch/bleve/index/upside_down/index_reader.go deleted file mode 100644 index b6043156..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/index_reader.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "bytes" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" -) - -type IndexReader struct { - index *UpsideDownCouch - kvreader store.KVReader - docCount uint64 -} - -func (i *IndexReader) TermFieldReader(term []byte, fieldName string) (index.TermFieldReader, error) { - fieldIndex, fieldExists := i.index.fieldIndexCache.FieldExists(fieldName) - if fieldExists { - return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex)) - } - return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0)) -} - -func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) { - return i.FieldDictRange(fieldName, nil, nil) -} - -func (i *IndexReader) FieldDictRange(fieldName string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { - fieldIndex, fieldExists := i.index.fieldIndexCache.FieldExists(fieldName) - if fieldExists { - return newUpsideDownCouchFieldDict(i, uint16(fieldIndex), startTerm, endTerm) - } - return newUpsideDownCouchFieldDict(i, ^uint16(0), []byte{ByteSeparator}, []byte{}) -} - -func (i *IndexReader) FieldDictPrefix(fieldName string, termPrefix []byte) (index.FieldDict, error) { - return i.FieldDictRange(fieldName, termPrefix, incrementBytes(termPrefix)) -} - -func (i *IndexReader) DocIDReader(start, end string) (index.DocIDReader, error) { - return newUpsideDownCouchDocIDReader(i, start, end) -} - -func (i *IndexReader) Document(id string) (doc *document.Document, err error) { - // first hit the back index to confirm doc exists - var backIndexRow *BackIndexRow - backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, id) - if err != nil { - return - } - if backIndexRow == nil { - return - } - doc = document.NewDocument(id) - storedRow := NewStoredRow(id, 0, []uint64{}, 'x', nil) - storedRowScanPrefix := storedRow.ScanPrefixForDoc() - it := i.kvreader.Iterator(storedRowScanPrefix) - defer func() { - if cerr := it.Close(); err == nil && cerr != nil { - err = cerr - } - }() - key, val, valid := it.Current() - for valid { - if !bytes.HasPrefix(key, storedRowScanPrefix) { - break - } - safeVal := val - if !i.kvreader.BytesSafeAfterClose() { - safeVal = make([]byte, len(val)) - copy(safeVal, val) - } - var row *StoredRow - row, err = NewStoredRowKV(key, safeVal) - if err != nil { - doc = nil - return - } - if row != nil { - fieldName := i.index.fieldIndexCache.FieldName(row.field) - field := decodeFieldType(row.typ, fieldName, row.value) - if field != nil { - doc.AddField(field) - } - } - - it.Next() - key, val, valid = it.Current() - } - return -} - -func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) { - back, err := i.index.backIndexRowForDoc(i.kvreader, id) - if err != nil { - return nil, err - } - rv := make(index.FieldTerms, len(back.termEntries)) - for _, entry := range back.termEntries { - fieldName := i.index.fieldIndexCache.FieldName(uint16(*entry.Field)) - terms, ok := rv[fieldName] - if !ok { - terms = make([]string, 0) - } - terms = append(terms, *entry.Term) - rv[fieldName] = terms - } - return rv, nil -} - -func (i *IndexReader) Fields() (fields []string, err error) { - fields = make([]string, 0) - it := i.kvreader.Iterator([]byte{'f'}) - defer func() { - if cerr := it.Close(); err == nil && cerr != nil { - err = cerr - } - }() - key, val, valid := it.Current() - for valid { - if !bytes.HasPrefix(key, []byte{'f'}) { - break - } - var row UpsideDownCouchRow - row, err = ParseFromKeyValue(key, val) - if err != nil { - fields = nil - return - } - if row != nil { - fieldRow, ok := row.(*FieldRow) - if ok { - fields = append(fields, fieldRow.name) - } - } - - it.Next() - key, val, valid = it.Current() - } - return -} - -func (i *IndexReader) GetInternal(key []byte) ([]byte, error) { - internalRow := NewInternalRow(key, nil) - return i.kvreader.Get(internalRow.Key()) -} - -func (i *IndexReader) DocCount() uint64 { - return i.docCount -} - -func (i *IndexReader) Close() error { - return i.kvreader.Close() -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/reader.go b/vendor/github.com/blevesearch/bleve/index/upside_down/reader.go deleted file mode 100644 index 967875bc..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/reader.go +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "bytes" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" -) - -type UpsideDownCouchTermFieldReader struct { - indexReader *IndexReader - iterator store.KVIterator - count uint64 - term []byte - field uint16 - readerPrefix []byte -} - -func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16) (*UpsideDownCouchTermFieldReader, error) { - dictionaryRow := NewDictionaryRow(term, field, 0) - val, err := indexReader.kvreader.Get(dictionaryRow.Key()) - if err != nil { - return nil, err - } - if val == nil { - return &UpsideDownCouchTermFieldReader{ - count: 0, - term: term, - field: field, - }, nil - } - - err = dictionaryRow.parseDictionaryV(val) - if err != nil { - return nil, err - } - - tfr := NewTermFrequencyRow(term, field, "", 0, 0) - readerPrefix := tfr.Key() - it := indexReader.kvreader.Iterator(readerPrefix) - - return &UpsideDownCouchTermFieldReader{ - indexReader: indexReader, - iterator: it, - count: dictionaryRow.count, - term: term, - field: field, - readerPrefix: readerPrefix, - }, nil -} - -func (r *UpsideDownCouchTermFieldReader) Count() uint64 { - return r.count -} - -func (r *UpsideDownCouchTermFieldReader) Next() (*index.TermFieldDoc, error) { - if r.iterator != nil { - key, val, valid := r.iterator.Current() - if valid { - if !bytes.HasPrefix(key, r.readerPrefix) { - // end of the line - return nil, nil - } - tfr, err := NewTermFrequencyRowKV(key, val) - if err != nil { - return nil, err - } - r.iterator.Next() - return &index.TermFieldDoc{ - ID: string(tfr.doc), - Freq: tfr.freq, - Norm: float64(tfr.norm), - Vectors: r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors), - }, nil - } - } - return nil, nil -} - -func (r *UpsideDownCouchTermFieldReader) Advance(docID string) (*index.TermFieldDoc, error) { - if r.iterator != nil { - tfr := NewTermFrequencyRow(r.term, r.field, docID, 0, 0) - r.iterator.Seek(tfr.Key()) - key, val, valid := r.iterator.Current() - if valid { - if !bytes.HasPrefix(key, r.readerPrefix) { - // end of the line - return nil, nil - } - tfr, err := NewTermFrequencyRowKV(key, val) - if err != nil { - return nil, err - } - r.iterator.Next() - return &index.TermFieldDoc{ - ID: string(tfr.doc), - Freq: tfr.freq, - Norm: float64(tfr.norm), - Vectors: r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors), - }, nil - } - } - return nil, nil -} - -func (r *UpsideDownCouchTermFieldReader) Close() error { - if r.iterator != nil { - return r.iterator.Close() - } - return nil -} - -type UpsideDownCouchDocIDReader struct { - indexReader *IndexReader - iterator store.KVIterator - start string - end string -} - -func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string) (*UpsideDownCouchDocIDReader, error) { - if start == "" { - start = string([]byte{0x0}) - } - if end == "" { - end = string([]byte{0xff}) - } - bisr := NewBackIndexRow(start, nil, nil) - it := indexReader.kvreader.Iterator(bisr.Key()) - - return &UpsideDownCouchDocIDReader{ - indexReader: indexReader, - iterator: it, - start: start, - end: end, - }, nil -} - -func (r *UpsideDownCouchDocIDReader) Next() (string, error) { - key, val, valid := r.iterator.Current() - if valid { - bier := NewBackIndexRow(r.end, nil, nil) - if bytes.Compare(key, bier.Key()) > 0 { - // end of the line - return "", nil - } - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return "", err - } - r.iterator.Next() - return string(br.doc), nil - } - return "", nil -} - -func (r *UpsideDownCouchDocIDReader) Advance(docID string) (string, error) { - bir := NewBackIndexRow(docID, nil, nil) - r.iterator.Seek(bir.Key()) - key, val, valid := r.iterator.Current() - if valid { - bier := NewBackIndexRow(r.end, nil, nil) - if bytes.Compare(key, bier.Key()) > 0 { - // end of the line - return "", nil - } - br, err := NewBackIndexRowKV(key, val) - if err != nil { - return "", err - } - r.iterator.Next() - return string(br.doc), nil - } - return "", nil -} - -func (r *UpsideDownCouchDocIDReader) Close() error { - return r.iterator.Close() -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/reader_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/reader_test.go deleted file mode 100644 index a27b6e2b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/reader_test.go +++ /dev/null @@ -1,297 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "os" - "reflect" - "testing" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store/boltdb" -) - -func TestIndexReader(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s := boltdb.New("test", "bleve") - s.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(s, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - doc = document.NewDocument("2") - doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer)) - doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // first look for a term that doesn't exist - reader, err := indexReader.TermFieldReader([]byte("nope"), "name") - if err != nil { - t.Errorf("Error accessing term field reader: %v", err) - } - count := reader.Count() - if count != 0 { - t.Errorf("Expected doc count to be: %d got: %d", 0, count) - } - err = reader.Close() - if err != nil { - t.Fatal(err) - } - - reader, err = indexReader.TermFieldReader([]byte("test"), "name") - if err != nil { - t.Errorf("Error accessing term field reader: %v", err) - } - - expectedCount = 2 - count = reader.Count() - if count != expectedCount { - t.Errorf("Exptected doc count to be: %d got: %d", expectedCount, count) - } - - var match *index.TermFieldDoc - var actualCount uint64 - match, err = reader.Next() - for err == nil && match != nil { - match, err = reader.Next() - if err != nil { - t.Errorf("unexpected error reading next") - } - actualCount++ - } - if actualCount != count { - t.Errorf("count was 2, but only saw %d", actualCount) - } - - expectedMatch := &index.TermFieldDoc{ - ID: "2", - Freq: 1, - Norm: 0.5773502588272095, - Vectors: []*index.TermFieldVector{ - &index.TermFieldVector{ - Field: "desc", - Pos: 3, - Start: 9, - End: 13, - }, - }, - } - tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - match, err = tfr.Next() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if !reflect.DeepEqual(expectedMatch, match) { - t.Errorf("got %#v, expected %#v", match, expectedMatch) - } - err = reader.Close() - if err != nil { - t.Fatal(err) - } - - // now test usage of advance - reader, err = indexReader.TermFieldReader([]byte("test"), "name") - if err != nil { - t.Errorf("Error accessing term field reader: %v", err) - } - - match, err = reader.Advance("2") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if match == nil { - t.Fatalf("Expected match, got nil") - } - if match.ID != "2" { - t.Errorf("Expected ID '2', got '%s'", match.ID) - } - match, err = reader.Advance("3") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if match != nil { - t.Errorf("expected nil, got %v", match) - } - err = reader.Close() - if err != nil { - t.Fatal(err) - } - - // now test creating a reader for a field that doesn't exist - reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist") - if err != nil { - t.Errorf("Error accessing term field reader: %v", err) - } - count = reader.Count() - if count != 0 { - t.Errorf("expected count 0 for reader of non-existant field") - } - match, err = reader.Next() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if match != nil { - t.Errorf("expected nil, got %v", match) - } - match, err = reader.Advance("anywhere") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if match != nil { - t.Errorf("expected nil, got %v", match) - } - -} - -func TestIndexDocIdReader(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - s := boltdb.New("test", "bleve") - s.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(s, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - doc = document.NewDocument("2") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test"))) - doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Error(err) - } - }() - - // first get all doc ids - reader, err := indexReader.DocIDReader("", "") - if err != nil { - t.Errorf("Error accessing doc id reader: %v", err) - } - defer func() { - err := reader.Close() - if err != nil { - t.Fatal(err) - } - }() - - id, err := reader.Next() - count := uint64(0) - for id != "" { - count++ - id, err = reader.Next() - } - if count != expectedCount { - t.Errorf("expected %d, got %d", expectedCount, count) - } - - // try it again, but jump to the second doc this time - reader2, err := indexReader.DocIDReader("", "") - if err != nil { - t.Errorf("Error accessing doc id reader: %v", err) - } - defer func() { - err := reader2.Close() - if err != nil { - t.Error(err) - } - }() - - id, err = reader2.Advance("2") - if err != nil { - t.Error(err) - } - if id != "2" { - t.Errorf("expected to find id '2', got '%s'", id) - } - - id, err = reader2.Advance("3") - if err != nil { - t.Error(err) - } - if id != "" { - t.Errorf("expected to find id '', got '%s'", id) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/row.go b/vendor/github.com/blevesearch/bleve/index/upside_down/row.go deleted file mode 100644 index d5cec28b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/row.go +++ /dev/null @@ -1,641 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "math" - - "github.com/golang/protobuf/proto" -) - -const ByteSeparator byte = 0xff - -type UpsideDownCouchRowStream chan UpsideDownCouchRow - -type UpsideDownCouchRow interface { - Key() []byte - Value() []byte -} - -func ParseFromKeyValue(key, value []byte) (UpsideDownCouchRow, error) { - if len(key) > 0 { - switch key[0] { - case 'v': - return NewVersionRowKV(key, value) - case 'f': - return NewFieldRowKV(key, value) - case 'd': - return NewDictionaryRowKV(key, value) - case 't': - return NewTermFrequencyRowKV(key, value) - case 'b': - return NewBackIndexRowKV(key, value) - case 's': - return NewStoredRowKV(key, value) - case 'i': - return NewInternalRowKV(key, value) - } - return nil, fmt.Errorf("Unknown field type '%s'", string(key[0])) - } - return nil, fmt.Errorf("Invalid empty key") -} - -// VERSION - -type VersionRow struct { - version uint8 -} - -func (v *VersionRow) Key() []byte { - return []byte{'v'} -} - -func (v *VersionRow) Value() []byte { - return []byte{byte(v.version)} -} - -func (v *VersionRow) String() string { - return fmt.Sprintf("Version: %d", v.version) -} - -func NewVersionRow(version uint8) *VersionRow { - return &VersionRow{ - version: version, - } -} - -func NewVersionRowKV(key, value []byte) (*VersionRow, error) { - rv := VersionRow{} - buf := bytes.NewBuffer(value) - err := binary.Read(buf, binary.LittleEndian, &rv.version) - if err != nil { - return nil, err - } - return &rv, nil -} - -// INTERNAL STORAGE - -type InternalRow struct { - key []byte - val []byte -} - -func (i *InternalRow) Key() []byte { - buf := make([]byte, len(i.key)+1) - buf[0] = 'i' - copy(buf[1:], i.key) - return buf -} - -func (i *InternalRow) Value() []byte { - return i.val -} - -func (i *InternalRow) String() string { - return fmt.Sprintf("InternalStore - Key: %s (% x) Val: %s (% x)", i.key, i.key, i.val, i.val) -} - -func NewInternalRow(key, val []byte) *InternalRow { - return &InternalRow{ - key: key, - val: val, - } -} - -func NewInternalRowKV(key, value []byte) (*InternalRow, error) { - rv := InternalRow{} - rv.key = key[1:] - rv.val = value - return &rv, nil -} - -// FIELD definition - -type FieldRow struct { - index uint16 - name string -} - -func (f *FieldRow) Key() []byte { - buf := make([]byte, 3) - buf[0] = 'f' - binary.LittleEndian.PutUint16(buf[1:3], f.index) - return buf -} - -func (f *FieldRow) Value() []byte { - return append([]byte(f.name), ByteSeparator) -} - -func (f *FieldRow) String() string { - return fmt.Sprintf("Field: %d Name: %s", f.index, f.name) -} - -func NewFieldRow(index uint16, name string) *FieldRow { - return &FieldRow{ - index: index, - name: name, - } -} - -func NewFieldRowKV(key, value []byte) (*FieldRow, error) { - rv := FieldRow{} - - buf := bytes.NewBuffer(key) - _, err := buf.ReadByte() // type - if err != nil { - return nil, err - } - err = binary.Read(buf, binary.LittleEndian, &rv.index) - if err != nil { - return nil, err - } - - buf = bytes.NewBuffer(value) - rv.name, err = buf.ReadString(ByteSeparator) - if err != nil { - return nil, err - } - rv.name = rv.name[:len(rv.name)-1] // trim off separator byte - - return &rv, nil -} - -// DICTIONARY - -type DictionaryRow struct { - field uint16 - term []byte - count uint64 -} - -func (dr *DictionaryRow) Key() []byte { - buf := make([]byte, 3+len(dr.term)) - buf[0] = 'd' - binary.LittleEndian.PutUint16(buf[1:3], dr.field) - copy(buf[3:], dr.term) - return buf -} - -func (dr *DictionaryRow) Value() []byte { - used := 0 - buf := make([]byte, 8) - - used += binary.PutUvarint(buf[used:used+8], dr.count) - - return buf[0:used] -} - -func (dr *DictionaryRow) String() string { - return fmt.Sprintf("Dictionary Term: `%s` Field: %d Count: %d ", string(dr.term), dr.field, dr.count) -} - -func NewDictionaryRow(term []byte, field uint16, count uint64) *DictionaryRow { - return &DictionaryRow{ - term: term, - field: field, - count: count, - } -} - -func NewDictionaryRowKV(key, value []byte) (*DictionaryRow, error) { - rv, err := NewDictionaryRowK(key) - if err != nil { - return nil, err - } - - err = rv.parseDictionaryV(value) - if err != nil { - return nil, err - } - return rv, nil - -} - -func NewDictionaryRowK(key []byte) (*DictionaryRow, error) { - rv := DictionaryRow{} - buf := bytes.NewBuffer(key) - _, err := buf.ReadByte() // type - if err != nil { - return nil, err - } - - err = binary.Read(buf, binary.LittleEndian, &rv.field) - if err != nil { - return nil, err - } - - rv.term, err = buf.ReadBytes(ByteSeparator) - // there is no separator expected here, should get EOF - if err != io.EOF { - return nil, err - } - - return &rv, nil -} - -func (dr *DictionaryRow) parseDictionaryV(value []byte) error { - buf := bytes.NewBuffer((value)) - - count, err := binary.ReadUvarint(buf) - if err != nil { - return err - } - dr.count = count - - return nil -} - -// TERM FIELD FREQUENCY - -type TermVector struct { - field uint16 - pos uint64 - start uint64 - end uint64 -} - -func (tv *TermVector) String() string { - return fmt.Sprintf("Field: %d Pos: %d Start: %d End %d", tv.field, tv.pos, tv.start, tv.end) -} - -type TermFrequencyRow struct { - term []byte - field uint16 - doc []byte - freq uint64 - norm float32 - vectors []*TermVector -} - -func (tfr *TermFrequencyRow) ScanPrefixForField() []byte { - buf := make([]byte, 3) - buf[0] = 't' - binary.LittleEndian.PutUint16(buf[1:3], tfr.field) - return buf -} - -func (tfr *TermFrequencyRow) ScanPrefixForFieldTermPrefix() []byte { - buf := make([]byte, 3+len(tfr.term)) - buf[0] = 't' - binary.LittleEndian.PutUint16(buf[1:3], tfr.field) - copy(buf[3:], tfr.term) - return buf -} - -func (tfr *TermFrequencyRow) ScanPrefixForFieldTerm() []byte { - buf := make([]byte, 3+len(tfr.term)+1) - buf[0] = 't' - binary.LittleEndian.PutUint16(buf[1:3], tfr.field) - termLen := copy(buf[3:], tfr.term) - buf[3+termLen] = ByteSeparator - return buf -} - -func (tfr *TermFrequencyRow) Key() []byte { - buf := make([]byte, 3+len(tfr.term)+1+len(tfr.doc)) - buf[0] = 't' - binary.LittleEndian.PutUint16(buf[1:3], tfr.field) - termLen := copy(buf[3:], tfr.term) - buf[3+termLen] = ByteSeparator - copy(buf[3+termLen+1:], tfr.doc) - return buf -} - -func (tfr *TermFrequencyRow) DictionaryRowKey() []byte { - dr := NewDictionaryRow(tfr.term, tfr.field, 0) - return dr.Key() -} - -func (tfr *TermFrequencyRow) Value() []byte { - used := 0 - buf := make([]byte, 8+8+(len(tfr.vectors)*(8+8+8+8))) - - used += binary.PutUvarint(buf[used:used+8], tfr.freq) - - normuint32 := math.Float32bits(tfr.norm) - newbuf := buf[used : used+8] - used += binary.PutUvarint(newbuf, uint64(normuint32)) - - for _, vector := range tfr.vectors { - used += binary.PutUvarint(buf[used:used+8], uint64(vector.field)) - used += binary.PutUvarint(buf[used:used+8], vector.pos) - used += binary.PutUvarint(buf[used:used+8], vector.start) - used += binary.PutUvarint(buf[used:used+8], vector.end) - } - return buf[0:used] -} - -func (tfr *TermFrequencyRow) String() string { - return fmt.Sprintf("Term: `%s` Field: %d DocId: `%s` Frequency: %d Norm: %f Vectors: %v", string(tfr.term), tfr.field, string(tfr.doc), tfr.freq, tfr.norm, tfr.vectors) -} - -func NewTermFrequencyRow(term []byte, field uint16, doc string, freq uint64, norm float32) *TermFrequencyRow { - return &TermFrequencyRow{ - term: term, - field: field, - doc: []byte(doc), - freq: freq, - norm: norm, - } -} - -func NewTermFrequencyRowWithTermVectors(term []byte, field uint16, doc string, freq uint64, norm float32, vectors []*TermVector) *TermFrequencyRow { - return &TermFrequencyRow{ - term: term, - field: field, - doc: []byte(doc), - freq: freq, - norm: norm, - vectors: vectors, - } -} - -func NewTermFrequencyRowK(key []byte) (*TermFrequencyRow, error) { - rv := TermFrequencyRow{} - keyLen := len(key) - if keyLen < 3 { - return nil, fmt.Errorf("invalid term frequency key, no valid field") - } - rv.field = binary.LittleEndian.Uint16(key[1:3]) - - termEndPos := bytes.IndexByte(key[3:], ByteSeparator) - if termEndPos < 0 { - return nil, fmt.Errorf("invalid term frequency key, no byte separator terminating term") - } - rv.term = key[3 : 3+termEndPos] - - docLen := len(key) - (3 + termEndPos + 1) - if docLen < 1 { - return nil, fmt.Errorf("invalid term frequency key, empty docid") - } - rv.doc = key[3+termEndPos+1:] - - return &rv, nil -} - -func (tfr *TermFrequencyRow) parseV(value []byte) error { - currOffset := 0 - bytesRead := 0 - tfr.freq, bytesRead = binary.Uvarint(value[currOffset:]) - if bytesRead <= 0 { - return fmt.Errorf("invalid term frequency value, invalid frequency") - } - currOffset += bytesRead - - var norm uint64 - norm, bytesRead = binary.Uvarint(value[currOffset:]) - if bytesRead <= 0 { - return fmt.Errorf("invalid term frequency value, no norm") - } - currOffset += bytesRead - - tfr.norm = math.Float32frombits(uint32(norm)) - - var field uint64 - field, bytesRead = binary.Uvarint(value[currOffset:]) - for bytesRead > 0 { - currOffset += bytesRead - tv := TermVector{} - tv.field = uint16(field) - // at this point we expect at least one term vector - if tfr.vectors == nil { - tfr.vectors = make([]*TermVector, 0) - } - - tv.pos, bytesRead = binary.Uvarint(value[currOffset:]) - if bytesRead <= 0 { - return fmt.Errorf("invalid term frequency value, vector contains no position") - } - currOffset += bytesRead - - tv.start, bytesRead = binary.Uvarint(value[currOffset:]) - if bytesRead <= 0 { - return fmt.Errorf("invalid term frequency value, vector contains no start") - } - currOffset += bytesRead - - tv.end, bytesRead = binary.Uvarint(value[currOffset:]) - if bytesRead <= 0 { - return fmt.Errorf("invalid term frequency value, vector contains no end") - } - currOffset += bytesRead - - tfr.vectors = append(tfr.vectors, &tv) - // try to read next record (may not exist) - field, bytesRead = binary.Uvarint(value[currOffset:]) - } - if len(value[currOffset:]) > 0 && bytesRead <= 0 { - return fmt.Errorf("invalid term frequency value, vector field invalid") - } - - return nil -} - -func NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) { - rv, err := NewTermFrequencyRowK(key) - if err != nil { - return nil, err - } - - err = rv.parseV(value) - if err != nil { - return nil, err - } - return rv, nil - -} - -type BackIndexRow struct { - doc []byte - termEntries []*BackIndexTermEntry - storedEntries []*BackIndexStoreEntry -} - -func (br *BackIndexRow) AllTermKeys() [][]byte { - if br == nil { - return nil - } - rv := make([][]byte, len(br.termEntries)) - for i, termEntry := range br.termEntries { - termRow := NewTermFrequencyRow([]byte(termEntry.GetTerm()), uint16(termEntry.GetField()), string(br.doc), 0, 0) - rv[i] = termRow.Key() - } - return rv -} - -func (br *BackIndexRow) AllStoredKeys() [][]byte { - if br == nil { - return nil - } - rv := make([][]byte, len(br.storedEntries)) - for i, storedEntry := range br.storedEntries { - storedRow := NewStoredRow(string(br.doc), uint16(storedEntry.GetField()), storedEntry.GetArrayPositions(), 'x', []byte{}) - rv[i] = storedRow.Key() - } - return rv -} - -func (br *BackIndexRow) Key() []byte { - buf := make([]byte, len(br.doc)+1) - buf[0] = 'b' - copy(buf[1:], br.doc) - return buf -} - -func (br *BackIndexRow) Value() []byte { - birv := &BackIndexRowValue{ - TermEntries: br.termEntries, - StoredEntries: br.storedEntries, - } - bytes, _ := proto.Marshal(birv) - return bytes -} - -func (br *BackIndexRow) String() string { - return fmt.Sprintf("Backindex DocId: `%s` Term Entries: %v, Stored Entries: %v", string(br.doc), br.termEntries, br.storedEntries) -} - -func NewBackIndexRow(doc string, entries []*BackIndexTermEntry, storedFields []*BackIndexStoreEntry) *BackIndexRow { - return &BackIndexRow{ - doc: []byte(doc), - termEntries: entries, - storedEntries: storedFields, - } -} - -func NewBackIndexRowKV(key, value []byte) (*BackIndexRow, error) { - rv := BackIndexRow{} - - buf := bytes.NewBuffer(key) - _, err := buf.ReadByte() // type - if err != nil { - return nil, err - } - - rv.doc, err = buf.ReadBytes(ByteSeparator) - if err == io.EOF && len(rv.doc) < 1 { - err = fmt.Errorf("invalid doc length 0") - } - if err != io.EOF { - return nil, err - } - - var birv BackIndexRowValue - err = proto.Unmarshal(value, &birv) - if err != nil { - return nil, err - } - rv.termEntries = birv.TermEntries - rv.storedEntries = birv.StoredEntries - - return &rv, nil -} - -// STORED - -type StoredRow struct { - doc []byte - field uint16 - arrayPositions []uint64 - typ byte - value []byte -} - -func (s *StoredRow) Key() []byte { - docLen := len(s.doc) - buf := make([]byte, 1+docLen+1+2+(binary.MaxVarintLen64*len(s.arrayPositions))) - buf[0] = 's' - copy(buf[1:], s.doc) - buf[1+docLen] = ByteSeparator - binary.LittleEndian.PutUint16(buf[1+docLen+1:], s.field) - bytesUsed := 1 + docLen + 1 + 2 - for _, arrayPosition := range s.arrayPositions { - varbytes := binary.PutUvarint(buf[bytesUsed:], arrayPosition) - bytesUsed += varbytes - } - return buf[0:bytesUsed] -} - -func (s *StoredRow) Value() []byte { - rv := make([]byte, len(s.value)+1) - rv[0] = s.typ - copy(rv[1:], s.value) - return rv -} - -func (s *StoredRow) String() string { - return fmt.Sprintf("Document: %s Field %d, Array Positions: %v, Type: %s Value: %s", s.doc, s.field, s.arrayPositions, string(s.typ), s.value) -} - -func (s *StoredRow) ScanPrefixForDoc() []byte { - docLen := len(s.doc) - buf := make([]byte, 1+docLen+1) - buf[0] = 's' - copy(buf[1:], s.doc) - buf[1+docLen] = ByteSeparator - return buf -} - -func NewStoredRow(doc string, field uint16, arrayPositions []uint64, typ byte, value []byte) *StoredRow { - return &StoredRow{ - doc: []byte(doc), - field: field, - arrayPositions: arrayPositions, - typ: typ, - value: value, - } -} - -func NewStoredRowK(key []byte) (*StoredRow, error) { - rv := StoredRow{} - - buf := bytes.NewBuffer(key) - _, err := buf.ReadByte() // type - if err != nil { - return nil, err - } - - rv.doc, err = buf.ReadBytes(ByteSeparator) - if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator - err = fmt.Errorf("invalid doc length 0") - return nil, err - } - - rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte - - err = binary.Read(buf, binary.LittleEndian, &rv.field) - if err != nil { - return nil, err - } - - rv.arrayPositions = make([]uint64, 0) - nextArrayPos, err := binary.ReadUvarint(buf) - for err == nil { - rv.arrayPositions = append(rv.arrayPositions, nextArrayPos) - nextArrayPos, err = binary.ReadUvarint(buf) - } - return &rv, nil -} - -func NewStoredRowKV(key, value []byte) (*StoredRow, error) { - rv, err := NewStoredRowK(key) - if err != nil { - return nil, err - } - rv.typ = value[0] - rv.value = value[1:] - return rv, nil -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/row_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/row_test.go deleted file mode 100644 index 98c7affa..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/row_test.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "reflect" - "testing" - - "github.com/golang/protobuf/proto" -) - -func TestRows(t *testing.T) { - tests := []struct { - input UpsideDownCouchRow - outKey []byte - outVal []byte - }{ - { - NewVersionRow(1), - []byte{'v'}, - []byte{0x1}, - }, - { - NewFieldRow(0, "name"), - []byte{'f', 0, 0}, - []byte{'n', 'a', 'm', 'e', ByteSeparator}, - }, - { - NewFieldRow(1, "desc"), - []byte{'f', 1, 0}, - []byte{'d', 'e', 's', 'c', ByteSeparator}, - }, - { - NewFieldRow(513, "style"), - []byte{'f', 1, 2}, - []byte{'s', 't', 'y', 'l', 'e', ByteSeparator}, - }, - { - NewDictionaryRow([]byte{'b', 'e', 'e', 'r'}, 0, 27), - []byte{'d', 0, 0, 'b', 'e', 'e', 'r'}, - []byte{27}, - }, - { - NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, "catz", 3, 3.14), - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'c', 'a', 't', 'z'}, - []byte{3, 195, 235, 163, 130, 4}, - }, - { - NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, "budweiser", 3, 3.14), - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{3, 195, 235, 163, 130, 4}, - }, - { - NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, "budweiser", 3, 3.14, []*TermVector{&TermVector{field: 0, pos: 1, start: 3, end: 11}, &TermVector{field: 0, pos: 2, start: 23, end: 31}, &TermVector{field: 0, pos: 3, start: 43, end: 51}}), - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 2, 23, 31, 0, 3, 43, 51}, - }, - // test larger varints - { - NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, "budweiser", 25896, 3.14, []*TermVector{&TermVector{field: 255, pos: 1, start: 3, end: 11}, &TermVector{field: 0, pos: 2198, start: 23, end: 31}, &TermVector{field: 0, pos: 3, start: 43, end: 51}}), - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 0, 150, 17, 23, 31, 0, 3, 43, 51}, - }, - { - NewBackIndexRow("budweiser", []*BackIndexTermEntry{&BackIndexTermEntry{Term: proto.String("beer"), Field: proto.Uint32(0)}}, nil), - []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{10, 8, 10, 4, 'b', 'e', 'e', 'r', 16, 0}, - }, - { - NewBackIndexRow("budweiser", []*BackIndexTermEntry{&BackIndexTermEntry{Term: proto.String("beer"), Field: proto.Uint32(0)}, &BackIndexTermEntry{Term: proto.String("beat"), Field: proto.Uint32(1)}}, nil), - []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{10, 8, 10, 4, 'b', 'e', 'e', 'r', 16, 0, 10, 8, 10, 4, 'b', 'e', 'a', 't', 16, 1}, - }, - { - NewBackIndexRow("budweiser", []*BackIndexTermEntry{&BackIndexTermEntry{Term: proto.String("beer"), Field: proto.Uint32(0)}, &BackIndexTermEntry{Term: proto.String("beat"), Field: proto.Uint32(1)}}, []*BackIndexStoreEntry{&BackIndexStoreEntry{Field: proto.Uint32(3)}, &BackIndexStoreEntry{Field: proto.Uint32(4)}, &BackIndexStoreEntry{Field: proto.Uint32(5)}}), - []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{10, 8, 10, 4, 'b', 'e', 'e', 'r', 16, 0, 10, 8, 10, 4, 'b', 'e', 'a', 't', 16, 1, 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5}, - }, - { - NewStoredRow("budweiser", 0, []uint64{}, byte('t'), []byte("an american beer")), - []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0}, - []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, - }, - { - NewStoredRow("budweiser", 0, []uint64{2, 294, 3078}, byte('t'), []byte("an american beer")), - []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0, 2, 166, 2, 134, 24}, - []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, - }, - { - NewInternalRow([]byte("mapping"), []byte(`{"mapping":"json content"}`)), - []byte{'i', 'm', 'a', 'p', 'p', 'i', 'n', 'g'}, - []byte{'{', '"', 'm', 'a', 'p', 'p', 'i', 'n', 'g', '"', ':', '"', 'j', 's', 'o', 'n', ' ', 'c', 'o', 'n', 't', 'e', 'n', 't', '"', '}'}, - }, - } - - // test going from struct to k/v bytes - for i, test := range tests { - rk := test.input.Key() - if !reflect.DeepEqual(rk, test.outKey) { - t.Errorf("Expected key to be %v got: %v", test.outKey, rk) - } - rv := test.input.Value() - if !reflect.DeepEqual(rv, test.outVal) { - t.Errorf("Expected value to be %v got: %v for %d", test.outVal, rv, i) - } - } - - // now test going back from k/v bytes to struct - for i, test := range tests { - row, err := ParseFromKeyValue(test.outKey, test.outVal) - if err != nil { - t.Errorf("error parsking key/value: %v", err) - } - if !reflect.DeepEqual(row, test.input) { - t.Errorf("Expected: %#v got: %#v for %d", test.input, row, i) - } - } - -} - -func TestInvalidRows(t *testing.T) { - tests := []struct { - key []byte - val []byte - }{ - // empty key - { - []byte{}, - []byte{}, - }, - // no such type q - { - []byte{'q'}, - []byte{}, - }, - // type v, invalid empty value - { - []byte{'v'}, - []byte{}, - }, - // type f, invalid key - { - []byte{'f'}, - []byte{}, - }, - // type f, valid key, invalid value - { - []byte{'f', 0, 0}, - []byte{}, - }, - // type t, invalid key (missing field) - { - []byte{'t'}, - []byte{}, - }, - // type t, invalid key (missing term) - { - []byte{'t', 0, 0}, - []byte{}, - }, - // type t, invalid key (missing id) - { - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator}, - []byte{}, - }, - // type t, invalid val (missing freq) - { - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{}, - }, - // type t, invalid val (missing norm) - { - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{3}, - }, - // type t, invalid val (half missing tv field, full missing is valid (no term vectors)) - { - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{3, 25, 255}, - }, - // type t, invalid val (missing tv pos) - { - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{3, 25, 0}, - }, - // type t, invalid val (missing tv start) - { - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{3, 25, 0, 0}, - }, - // type t, invalid val (missing tv end) - { - []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{3, 25, 0, 0, 0}, - }, - // type b, invalid key (missing id) - { - []byte{'b'}, - []byte{'b', 'e', 'e', 'r', ByteSeparator, 0, 0}, - }, - // type b, invalid val (missing field) - { - []byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}, - []byte{'g', 'a', 'r', 'b', 'a', 'g', 'e'}, - }, - // type s, invalid key (missing id) - { - []byte{'s'}, - []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, - }, - // type b, invalid val (missing field) - { - []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator}, - []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}, - }, - } - - for _, test := range tests { - _, err := ParseFromKeyValue(test.key, test.val) - if err == nil { - t.Errorf("expected error, got nil") - } - } -} - -func BenchmarkTermFrequencyRowEncode(b *testing.B) { - for i := 0; i < b.N; i++ { - row := NewTermFrequencyRowWithTermVectors( - []byte{'b', 'e', 'e', 'r'}, - 0, - "budweiser", - 3, - 3.14, - []*TermVector{ - &TermVector{ - field: 0, - pos: 1, - start: 3, - end: 11, - }, - &TermVector{ - field: 0, - pos: 2, - start: 23, - end: 31, - }, - &TermVector{ - field: 0, - pos: 3, - start: 43, - end: 51, - }, - }) - - row.Key() - row.Value() - } -} - -func BenchmarkTermFrequencyRowDecode(b *testing.B) { - for i := 0; i < b.N; i++ { - k := []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'} - v := []byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 2, 23, 31, 0, 3, 43, 51} - _, err := NewTermFrequencyRowKV(k, v) - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkBackIndexRowEncode(b *testing.B) { - field := uint32(1) - t1 := "term1" - for i := 0; i < b.N; i++ { - row := NewBackIndexRow("beername", - []*BackIndexTermEntry{ - &BackIndexTermEntry{ - Term: &t1, - Field: &field, - }, - }, - []*BackIndexStoreEntry{ - &BackIndexStoreEntry{ - Field: &field, - }, - }) - - row.Key() - row.Value() - } -} - -func BenchmarkBackIndexRowDecode(b *testing.B) { - for i := 0; i < b.N; i++ { - k := []byte{0x62, 0x62, 0x65, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65} - v := []byte{0x0a, 0x09, 0x0a, 0x05, 0x74, 0x65, 0x72, 0x6d, 0x31, 0x10, 0x01, 0x12, 0x02, 0x08, 0x01} - _, err := NewBackIndexRowKV(k, v) - if err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkStoredRowEncode(b *testing.B) { - for i := 0; i < b.N; i++ { - row := NewStoredRow("budweiser", 0, []uint64{}, byte('t'), []byte("an american beer")) - - row.Key() - row.Value() - } -} - -func BenchmarkStoredRowDecode(b *testing.B) { - for i := 0; i < b.N; i++ { - k := []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0} - v := []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'} - _, err := NewStoredRowKV(k, v) - if err != nil { - b.Fatal(err) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/stats.go b/vendor/github.com/blevesearch/bleve/index/upside_down/stats.go deleted file mode 100644 index e447b28b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/stats.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "encoding/json" - "sync/atomic" -) - -type indexStat struct { - updates, deletes, batches, errors uint64 - analysisTime, indexTime uint64 -} - -func (i *indexStat) MarshalJSON() ([]byte, error) { - m := map[string]interface{}{} - m["updates"] = atomic.LoadUint64(&i.updates) - m["deletes"] = atomic.LoadUint64(&i.deletes) - m["batches"] = atomic.LoadUint64(&i.batches) - m["errors"] = atomic.LoadUint64(&i.errors) - m["analysis_time"] = atomic.LoadUint64(&i.analysisTime) - m["index_time"] = atomic.LoadUint64(&i.indexTime) - return json.Marshal(m) -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.go b/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.go deleted file mode 100644 index ca1870b0..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.go +++ /dev/null @@ -1,721 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "bytes" - "encoding/json" - "fmt" - "math" - "sync/atomic" - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - - "github.com/golang/protobuf/proto" -) - -var VersionKey = []byte{'v'} - -const Version uint8 = 4 - -var IncompatibleVersion = fmt.Errorf("incompatible version, %d is supported", Version) - -type UpsideDownCouch struct { - version uint8 - path string - store store.KVStore - fieldIndexCache *FieldIndexCache - docCount uint64 - analysisQueue *AnalysisQueue - stats *indexStat -} - -func NewUpsideDownCouch(s store.KVStore, analysisQueue *AnalysisQueue) *UpsideDownCouch { - return &UpsideDownCouch{ - version: Version, - fieldIndexCache: NewFieldIndexCache(), - store: s, - analysisQueue: analysisQueue, - stats: &indexStat{}, - } -} - -func (udc *UpsideDownCouch) init(kvwriter store.KVWriter) (err error) { - // prepare a list of rows - rows := make([]UpsideDownCouchRow, 0) - - // version marker - rows = append(rows, NewVersionRow(udc.version)) - - return udc.batchRows(kvwriter, nil, rows, nil) -} - -func (udc *UpsideDownCouch) loadSchema(kvreader store.KVReader) (err error) { - - keyPrefix := []byte{'f'} - it := kvreader.Iterator(keyPrefix) - defer func() { - if cerr := it.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - it.Seek(keyPrefix) - key, val, valid := it.Current() - for valid { - - // stop when - if !bytes.HasPrefix(key, keyPrefix) { - break - } - var fieldRow *FieldRow - fieldRow, err = NewFieldRowKV(key, val) - if err != nil { - return - } - udc.fieldIndexCache.AddExisting(fieldRow.name, fieldRow.index) - - it.Next() - key, val, valid = it.Current() - } - - keyPrefix = []byte{'v'} - val, err = kvreader.Get(keyPrefix) - if err != nil { - return - } - var vr *VersionRow - vr, err = NewVersionRowKV(keyPrefix, val) - if err != nil { - return - } - if vr.version != Version { - err = IncompatibleVersion - return - } - - return -} - -func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) (err error) { - - // prepare batch - wb := writer.NewBatch() - - // add - for _, row := range addRows { - tfr, ok := row.(*TermFrequencyRow) - if ok { - // need to increment counter - dictionaryKey := tfr.DictionaryRowKey() - wb.Merge(dictionaryKey, dictionaryTermIncr) - } - wb.Set(row.Key(), row.Value()) - } - - // update - for _, row := range updateRows { - wb.Set(row.Key(), row.Value()) - } - - // delete - for _, row := range deleteRows { - tfr, ok := row.(*TermFrequencyRow) - if ok { - // need to decrement counter - dictionaryKey := tfr.DictionaryRowKey() - wb.Merge(dictionaryKey, dictionaryTermDecr) - } - wb.Delete(row.Key()) - } - - // write out the batch - err = wb.Execute() - if err != nil { - return - } - return -} - -func (udc *UpsideDownCouch) DocCount() (uint64, error) { - return udc.docCount, nil -} - -func (udc *UpsideDownCouch) Open() (err error) { - // install the merge operator - udc.store.SetMergeOperator(&mergeOperator) - - // now open the kv store - err = udc.store.Open() - if err != nil { - return - } - - // start a writer for the open process - var kvwriter store.KVWriter - kvwriter, err = udc.store.Writer() - if err != nil { - return - } - defer func() { - if cerr := kvwriter.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - var value []byte - value, err = kvwriter.Get(VersionKey) - if err != nil { - return - } - - // init new index OR load schema - if value == nil { - err = udc.init(kvwriter) - if err != nil { - return - } - } else { - err = udc.loadSchema(kvwriter) - if err != nil { - return - } - } - // set doc count - udc.docCount, err = udc.countDocs(kvwriter) - return -} - -func (udc *UpsideDownCouch) countDocs(kvreader store.KVReader) (count uint64, err error) { - it := kvreader.Iterator([]byte{'b'}) - defer func() { - if cerr := it.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - key, _, valid := it.Current() - for valid { - if !bytes.HasPrefix(key, []byte{'b'}) { - break - } - count++ - it.Next() - key, _, valid = it.Current() - } - - return -} - -func (udc *UpsideDownCouch) rowCount() (count uint64, err error) { - // start an isolated reader for use during the rowcount - kvreader, err := udc.store.Reader() - if err != nil { - return - } - defer func() { - if cerr := kvreader.Close(); err == nil && cerr != nil { - err = cerr - } - }() - it := kvreader.Iterator([]byte{0}) - defer func() { - if cerr := it.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - _, _, valid := it.Current() - for valid { - count++ - it.Next() - _, _, valid = it.Current() - } - - return -} - -func (udc *UpsideDownCouch) Close() error { - return udc.store.Close() -} - -func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { - // do analysis before acquiring write lock - analysisStart := time.Now() - resultChan := make(chan *AnalysisResult) - aw := AnalysisWork{ - udc: udc, - d: doc, - rc: resultChan, - } - // put the work on the queue - go func() { - udc.analysisQueue.Queue(&aw) - }() - - // wait for the result - result := <-resultChan - close(resultChan) - atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) - - // start a writer for this update - indexStart := time.Now() - var kvwriter store.KVWriter - kvwriter, err = udc.store.Writer() - if err != nil { - return - } - defer func() { - if cerr := kvwriter.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - // first we lookup the backindex row for the doc id if it exists - // lookup the back index row - var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvwriter, doc.ID) - if err != nil { - atomic.AddUint64(&udc.stats.errors, 1) - return - } - - // prepare a list of rows - addRows := make([]UpsideDownCouchRow, 0) - updateRows := make([]UpsideDownCouchRow, 0) - deleteRows := make([]UpsideDownCouchRow, 0) - - addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, result.rows, addRows, updateRows, deleteRows) - - err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows) - if err == nil && backIndexRow == nil { - udc.docCount++ - } - atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) - if err == nil { - atomic.AddUint64(&udc.stats.updates, 1) - } else { - atomic.AddUint64(&udc.stats.errors, 1) - } - return -} - -func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows, addRows, updateRows, deleteRows []UpsideDownCouchRow) ([]UpsideDownCouchRow, []UpsideDownCouchRow, []UpsideDownCouchRow) { - existingTermKeys := make(map[string]bool) - for _, key := range backIndexRow.AllTermKeys() { - existingTermKeys[string(key)] = true - } - - existingStoredKeys := make(map[string]bool) - for _, key := range backIndexRow.AllStoredKeys() { - existingStoredKeys[string(key)] = true - } - - for _, row := range rows { - switch row := row.(type) { - case *TermFrequencyRow: - rowKey := string(row.Key()) - if _, ok := existingTermKeys[rowKey]; ok { - updateRows = append(updateRows, row) - delete(existingTermKeys, rowKey) - } else { - addRows = append(addRows, row) - } - case *StoredRow: - rowKey := string(row.Key()) - if _, ok := existingStoredKeys[rowKey]; ok { - updateRows = append(updateRows, row) - delete(existingStoredKeys, rowKey) - } else { - addRows = append(addRows, row) - } - default: - updateRows = append(updateRows, row) - } - - } - - // any of the existing rows that weren't updated need to be deleted - for existingTermKey := range existingTermKeys { - termFreqRow, err := NewTermFrequencyRowK([]byte(existingTermKey)) - if err == nil { - deleteRows = append(deleteRows, termFreqRow) - } - } - - // any of the existing stored fields that weren't updated need to be deleted - for existingStoredKey := range existingStoredKeys { - storedRow, err := NewStoredRowK([]byte(existingStoredKey)) - if err == nil { - deleteRows = append(deleteRows, storedRow) - } - } - - return addRows, updateRows, deleteRows -} - -func (udc *UpsideDownCouch) storeField(docID string, field document.Field, fieldIndex uint16) ([]UpsideDownCouchRow, []*BackIndexStoreEntry) { - rows := make([]UpsideDownCouchRow, 0, 100) - backIndexStoredEntries := make([]*BackIndexStoreEntry, 0) - fieldType := encodeFieldType(field) - storedRow := NewStoredRow(docID, fieldIndex, field.ArrayPositions(), fieldType, field.Value()) - - // record the back index entry - backIndexStoredEntry := BackIndexStoreEntry{Field: proto.Uint32(uint32(fieldIndex)), ArrayPositions: field.ArrayPositions()} - backIndexStoredEntries = append(backIndexStoredEntries, &backIndexStoredEntry) - - rows = append(rows, storedRow) - return rows, backIndexStoredEntries -} - -func encodeFieldType(f document.Field) byte { - fieldType := byte('x') - switch f.(type) { - case *document.TextField: - fieldType = 't' - case *document.NumericField: - fieldType = 'n' - case *document.DateTimeField: - fieldType = 'd' - case *document.CompositeField: - fieldType = 'c' - } - return fieldType -} - -func (udc *UpsideDownCouch) indexField(docID string, field document.Field, fieldIndex uint16, fieldLength int, tokenFreqs analysis.TokenFrequencies) ([]UpsideDownCouchRow, []*BackIndexTermEntry) { - - rows := make([]UpsideDownCouchRow, 0, 100) - backIndexTermEntries := make([]*BackIndexTermEntry, 0) - fieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength))) - - for _, tf := range tokenFreqs { - var termFreqRow *TermFrequencyRow - if field.Options().IncludeTermVectors() { - tv, newFieldRows := udc.termVectorsFromTokenFreq(fieldIndex, tf) - rows = append(rows, newFieldRows...) - termFreqRow = NewTermFrequencyRowWithTermVectors(tf.Term, fieldIndex, docID, uint64(frequencyFromTokenFreq(tf)), fieldNorm, tv) - } else { - termFreqRow = NewTermFrequencyRow(tf.Term, fieldIndex, docID, uint64(frequencyFromTokenFreq(tf)), fieldNorm) - } - - // record the back index entry - backIndexTermEntry := BackIndexTermEntry{Term: proto.String(string(tf.Term)), Field: proto.Uint32(uint32(fieldIndex))} - backIndexTermEntries = append(backIndexTermEntries, &backIndexTermEntry) - - rows = append(rows, termFreqRow) - } - - return rows, backIndexTermEntries -} - -func (udc *UpsideDownCouch) Delete(id string) (err error) { - indexStart := time.Now() - // start a writer for this delete - var kvwriter store.KVWriter - kvwriter, err = udc.store.Writer() - if err != nil { - return - } - defer func() { - if cerr := kvwriter.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - // lookup the back index row - var backIndexRow *BackIndexRow - backIndexRow, err = udc.backIndexRowForDoc(kvwriter, id) - if err != nil { - atomic.AddUint64(&udc.stats.errors, 1) - return - } - if backIndexRow == nil { - atomic.AddUint64(&udc.stats.deletes, 1) - return - } - - deleteRows := make([]UpsideDownCouchRow, 0) - deleteRows = udc.deleteSingle(id, backIndexRow, deleteRows) - - err = udc.batchRows(kvwriter, nil, nil, deleteRows) - if err == nil { - udc.docCount-- - } - atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) - if err == nil { - atomic.AddUint64(&udc.stats.deletes, 1) - } else { - atomic.AddUint64(&udc.stats.errors, 1) - } - return -} - -func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, deleteRows []UpsideDownCouchRow) []UpsideDownCouchRow { - - for _, backIndexEntry := range backIndexRow.termEntries { - tfr := NewTermFrequencyRow([]byte(*backIndexEntry.Term), uint16(*backIndexEntry.Field), id, 0, 0) - deleteRows = append(deleteRows, tfr) - } - for _, se := range backIndexRow.storedEntries { - sf := NewStoredRow(id, uint16(*se.Field), se.ArrayPositions, 'x', nil) - deleteRows = append(deleteRows, sf) - } - - // also delete the back entry itself - deleteRows = append(deleteRows, backIndexRow) - return deleteRows -} - -func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID string) (*BackIndexRow, error) { - // use a temporary row structure to build key - tempRow := &BackIndexRow{ - doc: []byte(docID), - } - key := tempRow.Key() - value, err := kvreader.Get(key) - if err != nil { - return nil, err - } - if value == nil { - return nil, nil - } - backIndexRow, err := NewBackIndexRowKV(key, value) - if err != nil { - return nil, err - } - return backIndexRow, nil -} - -func (udc *UpsideDownCouch) backIndexRowsForBatch(kvreader store.KVReader, batch *index.Batch) (map[string]*BackIndexRow, error) { - // FIXME faster to order the ids and scan sequentially - // for now just get it working - rv := make(map[string]*BackIndexRow, 0) - for docID := range batch.IndexOps { - backIndexRow, err := udc.backIndexRowForDoc(kvreader, docID) - if err != nil { - return nil, err - } - rv[docID] = backIndexRow - } - return rv, nil -} - -func decodeFieldType(typ byte, name string, value []byte) document.Field { - switch typ { - case 't': - return document.NewTextField(name, []uint64{}, value) - case 'n': - return document.NewNumericFieldFromBytes(name, []uint64{}, value) - case 'd': - return document.NewDateTimeFieldFromBytes(name, []uint64{}, value) - } - return nil -} - -func frequencyFromTokenFreq(tf *analysis.TokenFreq) int { - return len(tf.Locations) -} - -func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *analysis.TokenFreq) ([]*TermVector, []UpsideDownCouchRow) { - rv := make([]*TermVector, len(tf.Locations)) - newFieldRows := make([]UpsideDownCouchRow, 0) - - for i, l := range tf.Locations { - var newFieldRow *FieldRow - fieldIndex := field - if l.Field != "" { - // lookup correct field - fieldIndex, newFieldRow = udc.fieldIndexCache.FieldIndex(l.Field) - if newFieldRow != nil { - newFieldRows = append(newFieldRows, newFieldRow) - } - } - tv := TermVector{ - field: fieldIndex, - pos: uint64(l.Position), - start: uint64(l.Start), - end: uint64(l.End), - } - rv[i] = &tv - } - - return rv, newFieldRows -} - -func (udc *UpsideDownCouch) termFieldVectorsFromTermVectors(in []*TermVector) []*index.TermFieldVector { - rv := make([]*index.TermFieldVector, len(in)) - - for i, tv := range in { - fieldName := udc.fieldIndexCache.FieldName(tv.field) - tfv := index.TermFieldVector{ - Field: fieldName, - Pos: tv.pos, - Start: tv.start, - End: tv.end, - } - rv[i] = &tfv - } - return rv -} - -func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { - analysisStart := time.Now() - resultChan := make(chan *AnalysisResult) - - var numUpdates uint64 - for _, doc := range batch.IndexOps { - if doc != nil { - numUpdates++ - } - } - - go func() { - for _, doc := range batch.IndexOps { - if doc != nil { - aw := AnalysisWork{ - udc: udc, - d: doc, - rc: resultChan, - } - // put the work on the queue - udc.analysisQueue.Queue(&aw) - } - } - }() - - newRowsMap := make(map[string][]UpsideDownCouchRow) - // wait for the result - var itemsDeQueued uint64 - for itemsDeQueued < numUpdates { - result := <-resultChan - newRowsMap[result.docID] = result.rows - itemsDeQueued++ - } - close(resultChan) - - atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) - - indexStart := time.Now() - // start a writer for this batch - var kvwriter store.KVWriter - kvwriter, err = udc.store.Writer() - if err != nil { - return - } - defer func() { - if cerr := kvwriter.Close(); err == nil && cerr != nil { - err = cerr - } - }() - - // first lookup all the back index rows - var backIndexRows map[string]*BackIndexRow - backIndexRows, err = udc.backIndexRowsForBatch(kvwriter, batch) - if err != nil { - return - } - - // prepare a list of rows - addRows := make([]UpsideDownCouchRow, 0) - updateRows := make([]UpsideDownCouchRow, 0) - deleteRows := make([]UpsideDownCouchRow, 0) - - docsAdded := uint64(0) - docsDeleted := uint64(0) - for docID, doc := range batch.IndexOps { - backIndexRow := backIndexRows[docID] - if doc == nil && backIndexRow != nil { - // delete - deleteRows = udc.deleteSingle(docID, backIndexRow, deleteRows) - docsDeleted++ - } else if doc != nil { - addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, newRowsMap[docID], addRows, updateRows, deleteRows) - if backIndexRow == nil { - docsAdded++ - } - } - } - - // add the internal ops - for internalKey, internalValue := range batch.InternalOps { - if internalValue == nil { - // delete - deleteInternalRow := NewInternalRow([]byte(internalKey), nil) - deleteRows = append(deleteRows, deleteInternalRow) - } else { - updateInternalRow := NewInternalRow([]byte(internalKey), internalValue) - updateRows = append(updateRows, updateInternalRow) - } - } - - err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows) - atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) - if err == nil { - udc.docCount += docsAdded - udc.docCount -= docsDeleted - atomic.AddUint64(&udc.stats.updates, numUpdates) - atomic.AddUint64(&udc.stats.deletes, docsDeleted) - atomic.AddUint64(&udc.stats.batches, 1) - } else { - atomic.AddUint64(&udc.stats.errors, 1) - } - return -} - -func (udc *UpsideDownCouch) SetInternal(key, val []byte) (err error) { - internalRow := NewInternalRow(key, val) - var writer store.KVWriter - writer, err = udc.store.Writer() - if err != nil { - return - } - defer func() { - if cerr := writer.Close(); err == nil && cerr != nil { - err = cerr - } - }() - return writer.Set(internalRow.Key(), internalRow.Value()) -} - -func (udc *UpsideDownCouch) DeleteInternal(key []byte) (err error) { - internalRow := NewInternalRow(key, nil) - var writer store.KVWriter - writer, err = udc.store.Writer() - if err != nil { - return - } - defer func() { - if cerr := writer.Close(); err == nil && cerr != nil { - err = cerr - } - }() - return writer.Delete(internalRow.Key()) -} - -func (udc *UpsideDownCouch) Reader() (index.IndexReader, error) { - kvr, err := udc.store.Reader() - if err != nil { - return nil, fmt.Errorf("error opening store reader: %v", err) - } - return &IndexReader{ - index: udc, - kvreader: kvr, - docCount: udc.docCount, - }, nil -} - -func (udc *UpsideDownCouch) Stats() json.Marshaler { - return udc.stats -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.pb.go b/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.pb.go deleted file mode 100644 index f35efab1..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.pb.go +++ /dev/null @@ -1,98 +0,0 @@ -// Code generated by protoc-gen-go. -// source: upside_down.proto -// DO NOT EDIT! - -/* -Package upside_down is a generated protocol buffer package. - -It is generated from these files: - upside_down.proto - -It has these top-level messages: - BackIndexTermEntry - BackIndexStoreEntry - BackIndexRowValue -*/ -package upside_down - -import proto "github.com/golang/protobuf/proto" -import math "math" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = math.Inf - -type BackIndexTermEntry struct { - Term *string `protobuf:"bytes,1,req,name=term" json:"term,omitempty"` - Field *uint32 `protobuf:"varint,2,req,name=field" json:"field,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *BackIndexTermEntry) Reset() { *m = BackIndexTermEntry{} } -func (m *BackIndexTermEntry) String() string { return proto.CompactTextString(m) } -func (*BackIndexTermEntry) ProtoMessage() {} - -func (m *BackIndexTermEntry) GetTerm() string { - if m != nil && m.Term != nil { - return *m.Term - } - return "" -} - -func (m *BackIndexTermEntry) GetField() uint32 { - if m != nil && m.Field != nil { - return *m.Field - } - return 0 -} - -type BackIndexStoreEntry struct { - Field *uint32 `protobuf:"varint,1,req,name=field" json:"field,omitempty"` - ArrayPositions []uint64 `protobuf:"varint,2,rep,name=arrayPositions" json:"arrayPositions,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *BackIndexStoreEntry) Reset() { *m = BackIndexStoreEntry{} } -func (m *BackIndexStoreEntry) String() string { return proto.CompactTextString(m) } -func (*BackIndexStoreEntry) ProtoMessage() {} - -func (m *BackIndexStoreEntry) GetField() uint32 { - if m != nil && m.Field != nil { - return *m.Field - } - return 0 -} - -func (m *BackIndexStoreEntry) GetArrayPositions() []uint64 { - if m != nil { - return m.ArrayPositions - } - return nil -} - -type BackIndexRowValue struct { - TermEntries []*BackIndexTermEntry `protobuf:"bytes,1,rep,name=termEntries" json:"termEntries,omitempty"` - StoredEntries []*BackIndexStoreEntry `protobuf:"bytes,2,rep,name=storedEntries" json:"storedEntries,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *BackIndexRowValue) Reset() { *m = BackIndexRowValue{} } -func (m *BackIndexRowValue) String() string { return proto.CompactTextString(m) } -func (*BackIndexRowValue) ProtoMessage() {} - -func (m *BackIndexRowValue) GetTermEntries() []*BackIndexTermEntry { - if m != nil { - return m.TermEntries - } - return nil -} - -func (m *BackIndexRowValue) GetStoredEntries() []*BackIndexStoreEntry { - if m != nil { - return m.StoredEntries - } - return nil -} - -func init() { -} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down_test.go b/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down_test.go deleted file mode 100644 index 31d16c0b..00000000 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down_test.go +++ /dev/null @@ -1,1177 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package upside_down - -import ( - "os" - "reflect" - "regexp" - "testing" - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store/boltdb" -) - -var testAnalyzer = &analysis.Analyzer{ - Tokenizer: regexp_tokenizer.NewRegexpTokenizer(regexp.MustCompile(`\w+`)), -} - -func TestIndexOpenReopen(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - - var expectedCount uint64 - docCount, err := idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - // opening the database should have inserted a version - expectedLength := uint64(1) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } - - // now close it - err = idx.Close() - if err != nil { - t.Fatal(err) - } - - store = boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - idx = NewUpsideDownCouch(store, analysisQueue) - err = idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - - // now close it - err = idx.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestIndexInsert(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - docCount, err := idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - docCount, err = idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - // should have 4 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the term count, and 1 for the back index entry) - expectedLength := uint64(1 + 1 + 1 + 1 + 1) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } -} - -func TestIndexInsertThenDelete(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - docCount, err := idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - doc2 := document.NewDocument("2") - doc2.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc2) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - docCount, err = idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - err = idx.Delete("1") - if err != nil { - t.Errorf("Error deleting entry from index: %v", err) - } - expectedCount-- - - docCount, err = idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - err = idx.Delete("2") - if err != nil { - t.Errorf("Error deleting entry from index: %v", err) - } - expectedCount-- - - docCount, err = idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - // should have 2 rows (1 for version, 1 for schema field, 1 for dictionary row garbage) - expectedLength := uint64(1 + 1 + 1) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } -} - -func TestIndexInsertThenUpdate(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - // this update should overwrite one term, and introduce one new one - doc = document.NewDocument("1") - doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test fail"), testAnalyzer)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error deleting entry from index: %v", err) - } - - // should have 2 rows (1 for version, 1 for schema field, and 2 for the two term, and 2 for the term counts, and 1 for the back index entry) - expectedLength := uint64(1 + 1 + 2 + 2 + 1) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } - - // now do another update that should remove one of the terms - doc = document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("fail"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error deleting entry from index: %v", err) - } - - // should have 2 rows (1 for version, 1 for schema field, and 1 for the remaining term, and 2 for the term diciontary, and 1 for the back index entry) - expectedLength = uint64(1 + 1 + 1 + 2 + 1) - rowCount, err = idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } -} - -func TestIndexInsertMultiple(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - - var expectedCount uint64 - - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - doc = document.NewDocument("2") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - // should have 4 rows (1 for version, 1 for schema field, and 2 for single term, and 1 for the term count, and 2 for the back index entries) - expectedLength := uint64(1 + 1 + 2 + 1 + 2) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } - - // close, reopen and add one more to test that counting works correctly - err = idx.Close() - if err != nil { - t.Fatal(err) - } - - store = boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - idx = NewUpsideDownCouch(store, analysisQueue) - err = idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - doc = document.NewDocument("3") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - docCount, err := idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("expected doc count: %d, got %d", expectedCount, docCount) - } -} - -func TestIndexInsertWithStore(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - docCount, err := idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - docCount, err = idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - // should have 6 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the stored field and 1 for the term count, and 1 for the back index entry) - expectedLength := uint64(1 + 1 + 1 + 1 + 1 + 1) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - storedDoc, err := indexReader.Document("1") - if err != nil { - t.Error(err) - } - - if len(storedDoc.Fields) != 1 { - t.Errorf("expected 1 stored field, got %d", len(storedDoc.Fields)) - } - textField, ok := storedDoc.Fields[0].(*document.TextField) - if !ok { - t.Errorf("expected text field") - } - if string(textField.Value()) != "test" { - t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) - } -} - -func TestIndexInternalCRUD(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // get something that doesn't exist yet - val, err := indexReader.GetInternal([]byte("key")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %s", val) - } - - // set - err = idx.SetInternal([]byte("key"), []byte("abc")) - if err != nil { - t.Error(err) - } - - indexReader2, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader2.Close() - if err != nil { - t.Fatal(err) - } - }() - - // get - val, err = indexReader2.GetInternal([]byte("key")) - if err != nil { - t.Error(err) - } - if string(val) != "abc" { - t.Errorf("expected %s, got '%s'", "abc", val) - } - - // delete - err = idx.DeleteInternal([]byte("key")) - if err != nil { - t.Error(err) - } - - indexReader3, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader3.Close() - if err != nil { - t.Fatal(err) - } - }() - - // get again - val, err = indexReader3.GetInternal([]byte("key")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got %s", val) - } -} - -func TestIndexBatch(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - - // first create 2 docs the old fashioned way - doc := document.NewDocument("1") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - doc = document.NewDocument("2") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2"))) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - // now create a batch which does 3 things - // insert new doc - // update existing doc - // delete existing doc - // net document count change 0 - - batch := index.NewBatch() - doc = document.NewDocument("3") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) - batch.Update(doc) - doc = document.NewDocument("2") - doc.AddField(document.NewTextField("name", []uint64{}, []byte("test2updated"))) - batch.Update(doc) - batch.Delete("1") - - err = idx.Batch(batch) - if err != nil { - t.Error(err) - } - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - docCount := indexReader.DocCount() - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - docIDReader, err := indexReader.DocIDReader("", "") - if err != nil { - t.Error(err) - } - docIds := make([]string, 0) - docID, err := docIDReader.Next() - for docID != "" && err == nil { - docIds = append(docIds, docID) - docID, err = docIDReader.Next() - } - if err != nil { - t.Error(err) - } - expectedDocIds := []string{"2", "3"} - if !reflect.DeepEqual(docIds, expectedDocIds) { - t.Errorf("expected ids: %v, got ids: %v", expectedDocIds, docIds) - } -} - -func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - var expectedCount uint64 - docCount, err := idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) - doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) - df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) - if err != nil { - t.Error(err) - } - doc.AddField(df) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - expectedCount++ - - docCount, err = idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - // should have 72 rows - // 1 for version - // 3 for schema fields - // 1 for text term - // 16 for numeric terms - // 16 for date terms - // 3 for the stored field - // 1 for the text term count - // 16 for numeric term counts - // 16 for date term counts - // 1 for the back index entry - expectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Error(err) - } - }() - - storedDoc, err := indexReader.Document("1") - if err != nil { - t.Error(err) - } - - if len(storedDoc.Fields) != 3 { - t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) - } - textField, ok := storedDoc.Fields[0].(*document.TextField) - if !ok { - t.Errorf("expected text field") - } - if string(textField.Value()) != "test" { - t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) - } - numField, ok := storedDoc.Fields[1].(*document.NumericField) - if !ok { - t.Errorf("expected numeric field") - } - numFieldNumer, err := numField.Number() - if err != nil { - t.Error(err) - } else { - if numFieldNumer != 35.99 { - t.Errorf("expeted numeric value 35.99, got %f", numFieldNumer) - } - } - dateField, ok := storedDoc.Fields[2].(*document.DateTimeField) - if !ok { - t.Errorf("expected date field") - } - dateFieldDate, err := dateField.DateTime() - if err != nil { - t.Error(err) - } else { - if dateFieldDate != time.Unix(0, 0).UTC() { - t.Errorf("expected date value unix epoch, got %v", dateFieldDate) - } - } - - // now update the document, but omit one of the fields - doc = document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField)) - doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - indexReader2, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader2.Close() - if err != nil { - t.Error(err) - } - }() - - // expected doc count shouldn't have changed - docCount = indexReader2.DocCount() - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } - - // should only get 2 fields back now though - storedDoc, err = indexReader2.Document("1") - if err != nil { - t.Error(err) - } - - if len(storedDoc.Fields) != 2 { - t.Errorf("expected 3 stored field, got %d", len(storedDoc.Fields)) - } - textField, ok = storedDoc.Fields[0].(*document.TextField) - if !ok { - t.Errorf("expected text field") - } - if string(textField.Value()) != "testup" { - t.Errorf("expected field content 'testup', got '%s'", string(textField.Value())) - } - numField, ok = storedDoc.Fields[1].(*document.NumericField) - if !ok { - t.Errorf("expected numeric field") - } - numFieldNumer, err = numField.Number() - if err != nil { - t.Error(err) - } else { - if numFieldNumer != 36.99 { - t.Errorf("expeted numeric value 36.99, got %f", numFieldNumer) - } - } - - // now delete the document - err = idx.Delete("1") - expectedCount-- - - // expected doc count shouldn't have changed - docCount, err = idx.DocCount() - if err != nil { - t.Error(err) - } - if docCount != expectedCount { - t.Errorf("Expected document count to be %d got %d", expectedCount, docCount) - } -} - -func TestIndexInsertFields(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) - doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField)) - dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField) - if err != nil { - t.Error(err) - } - doc.AddField(dateField) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - fields, err := indexReader.Fields() - if err != nil { - t.Error(err) - } else { - expectedFields := []string{"name", "age", "unixEpoch"} - if !reflect.DeepEqual(fields, expectedFields) { - t.Errorf("expected fields: %v, got %v", expectedFields, fields) - } - } - -} - -func TestIndexUpdateComposites(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) - doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) - doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - // should have 72 rows - // 1 for version - // 3 for schema fields - // 4 for text term - // 2 for the stored field - // 4 for the text term count - // 1 for the back index entry - expectedLength := uint64(1 + 3 + 4 + 2 + 4 + 1) - rowCount, err := idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } - - // now lets update it - doc = document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField)) - doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField)) - doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // make sure new values are in index - storedDoc, err := indexReader.Document("1") - if err != nil { - t.Error(err) - } - if len(storedDoc.Fields) != 2 { - t.Errorf("expected 2 stored field, got %d", len(storedDoc.Fields)) - } - textField, ok := storedDoc.Fields[0].(*document.TextField) - if !ok { - t.Errorf("expected text field") - } - if string(textField.Value()) != "testupdated" { - t.Errorf("expected field content 'test', got '%s'", string(textField.Value())) - } - - // should have the same row count as before, plus 4 term dictionary garbage rows - expectedLength += 4 - rowCount, err = idx.rowCount() - if err != nil { - t.Error(err) - } - if rowCount != expectedLength { - t.Errorf("expected %d rows, got: %d", expectedLength, rowCount) - } -} - -func TestIndexFieldsMisc(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField)) - doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - fieldName1 := idx.fieldIndexCache.FieldName(1) - if fieldName1 != "name" { - t.Errorf("expected field named 'name', got '%s'", fieldName1) - } - fieldName2 := idx.fieldIndexCache.FieldName(2) - if fieldName2 != "title" { - t.Errorf("expected field named 'title', got '%s'", fieldName2) - } - fieldName3 := idx.fieldIndexCache.FieldName(3) - if fieldName3 != "" { - t.Errorf("expected field named '', got '%s'", fieldName3) - } - -} - -func TestIndexTermReaderCompositeFields(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)) - doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors)) - doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all") - if err != nil { - t.Error(err) - } - - tfd, err := termFieldReader.Next() - for tfd != nil && err == nil { - if tfd.ID != "1" { - t.Errorf("expected to find document id 1") - } - tfd, err = termFieldReader.Next() - } - if err != nil { - t.Error(err) - } -} - -func TestIndexDocumentFieldTerms(t *testing.T) { - defer func() { - err := os.RemoveAll("test") - if err != nil { - t.Fatal(err) - } - }() - - store := boltdb.New("test", "bleve") - store.SetMergeOperator(&mergeOperator) - analysisQueue := NewAnalysisQueue(1) - idx := NewUpsideDownCouch(store, analysisQueue) - err := idx.Open() - if err != nil { - t.Errorf("error opening index: %v", err) - } - defer func() { - err := idx.Close() - if err != nil { - t.Fatal(err) - } - }() - - doc := document.NewDocument("1") - doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors)) - doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors)) - err = idx.Update(doc) - if err != nil { - t.Errorf("Error updating index: %v", err) - } - - indexReader, err := idx.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - fieldTerms, err := indexReader.DocumentFieldTerms("1") - if err != nil { - t.Error(err) - } - expectedFieldTerms := index.FieldTerms{ - "name": []string{"test"}, - "title": []string{"mister"}, - } - if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) { - t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms) - } -} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go new file mode 100644 index 00000000..d1b1fd59 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/analysis.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" +) + +func (udc *UpsideDownCouch) Analyze(d *document.Document) *index.AnalysisResult { + rv := &index.AnalysisResult{ + DocID: d.ID, + Rows: make([]index.IndexRow, 0, 100), + } + + docIDBytes := []byte(d.ID) + + // track our back index entries + backIndexStoredEntries := make([]*BackIndexStoreEntry, 0) + + // information we collate as we merge fields with same name + fieldTermFreqs := make(map[uint16]analysis.TokenFrequencies) + fieldLengths := make(map[uint16]int) + fieldIncludeTermVectors := make(map[uint16]bool) + fieldNames := make(map[uint16]string) + + analyzeField := func(field document.Field, storable bool) { + fieldIndex, newFieldRow := udc.fieldIndexOrNewRow(field.Name()) + if newFieldRow != nil { + rv.Rows = append(rv.Rows, newFieldRow) + } + fieldNames[fieldIndex] = field.Name() + + if field.Options().IsIndexed() { + fieldLength, tokenFreqs := field.Analyze() + existingFreqs := fieldTermFreqs[fieldIndex] + if existingFreqs == nil { + fieldTermFreqs[fieldIndex] = tokenFreqs + } else { + existingFreqs.MergeAll(field.Name(), tokenFreqs) + fieldTermFreqs[fieldIndex] = existingFreqs + } + fieldLengths[fieldIndex] += fieldLength + fieldIncludeTermVectors[fieldIndex] = field.Options().IncludeTermVectors() + } + + if storable && field.Options().IsStored() { + rv.Rows, backIndexStoredEntries = udc.storeField(docIDBytes, field, fieldIndex, rv.Rows, backIndexStoredEntries) + } + } + + // walk all the fields, record stored fields now + // place information about indexed fields into map + // this collates information across fields with + // same names (arrays) + for _, field := range d.Fields { + analyzeField(field, true) + } + + if len(d.CompositeFields) > 0 { + for fieldIndex, tokenFreqs := range fieldTermFreqs { + // see if any of the composite fields need this + for _, compositeField := range d.CompositeFields { + compositeField.Compose(fieldNames[fieldIndex], fieldLengths[fieldIndex], tokenFreqs) + } + } + + for _, compositeField := range d.CompositeFields { + analyzeField(compositeField, false) + } + } + + rowsCapNeeded := len(rv.Rows) + 1 + for _, tokenFreqs := range fieldTermFreqs { + rowsCapNeeded += len(tokenFreqs) + } + + rv.Rows = append(make([]index.IndexRow, 0, rowsCapNeeded), rv.Rows...) + + backIndexTermsEntries := make([]*BackIndexTermsEntry, 0, len(fieldTermFreqs)) + + // walk through the collated information and process + // once for each indexed field (unique name) + for fieldIndex, tokenFreqs := range fieldTermFreqs { + fieldLength := fieldLengths[fieldIndex] + includeTermVectors := fieldIncludeTermVectors[fieldIndex] + + // encode this field + rv.Rows, backIndexTermsEntries = udc.indexField(docIDBytes, includeTermVectors, fieldIndex, fieldLength, tokenFreqs, rv.Rows, backIndexTermsEntries) + } + + // build the back index row + backIndexRow := NewBackIndexRow(docIDBytes, backIndexTermsEntries, backIndexStoredEntries) + rv.Rows = append(rv.Rows, backIndexRow) + + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_all.sh b/vendor/github.com/blevesearch/bleve/index/upsidedown/benchmark_all.sh old mode 100644 new mode 100755 similarity index 100% rename from vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_all.sh rename to vendor/github.com/blevesearch/bleve/index/upsidedown/benchmark_all.sh diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go new file mode 100644 index 00000000..cb045d24 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/dump.go @@ -0,0 +1,174 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "sort" + + "github.com/blevesearch/bleve/index/store" +) + +// the functions in this file are only intended to be used by +// the bleve_dump utility and the debug http handlers +// if your application relies on them, you're doing something wrong +// they may change or be removed at any time + +func dumpPrefix(kvreader store.KVReader, rv chan interface{}, prefix []byte) { + start := prefix + if start == nil { + start = []byte{0} + } + it := kvreader.PrefixIterator(start) + defer func() { + cerr := it.Close() + if cerr != nil { + rv <- cerr + } + }() + key, val, valid := it.Current() + for valid { + ck := make([]byte, len(key)) + copy(ck, key) + cv := make([]byte, len(val)) + copy(cv, val) + row, err := ParseFromKeyValue(ck, cv) + if err != nil { + rv <- err + return + } + rv <- row + + it.Next() + key, val, valid = it.Current() + } +} + +func dumpRange(kvreader store.KVReader, rv chan interface{}, start, end []byte) { + it := kvreader.RangeIterator(start, end) + defer func() { + cerr := it.Close() + if cerr != nil { + rv <- cerr + } + }() + key, val, valid := it.Current() + for valid { + ck := make([]byte, len(key)) + copy(ck, key) + cv := make([]byte, len(val)) + copy(cv, val) + row, err := ParseFromKeyValue(ck, cv) + if err != nil { + rv <- err + return + } + rv <- row + + it.Next() + key, val, valid = it.Current() + } +} + +func (i *IndexReader) DumpAll() chan interface{} { + rv := make(chan interface{}) + go func() { + defer close(rv) + dumpRange(i.kvreader, rv, nil, nil) + }() + return rv +} + +func (i *IndexReader) DumpFields() chan interface{} { + rv := make(chan interface{}) + go func() { + defer close(rv) + dumpPrefix(i.kvreader, rv, []byte{'f'}) + }() + return rv +} + +type keyset [][]byte + +func (k keyset) Len() int { return len(k) } +func (k keyset) Swap(i, j int) { k[i], k[j] = k[j], k[i] } +func (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 } + +// DumpDoc returns all rows in the index related to this doc id +func (i *IndexReader) DumpDoc(id string) chan interface{} { + idBytes := []byte(id) + + rv := make(chan interface{}) + + go func() { + defer close(rv) + + back, err := backIndexRowForDoc(i.kvreader, []byte(id)) + if err != nil { + rv <- err + return + } + + // no such doc + if back == nil { + return + } + // build sorted list of term keys + keys := make(keyset, 0) + for _, entry := range back.termsEntries { + for i := range entry.Terms { + tfr := NewTermFrequencyRow([]byte(entry.Terms[i]), uint16(*entry.Field), idBytes, 0, 0) + key := tfr.Key() + keys = append(keys, key) + } + } + sort.Sort(keys) + + // first add all the stored rows + storedRowPrefix := NewStoredRow(idBytes, 0, []uint64{}, 'x', []byte{}).ScanPrefixForDoc() + dumpPrefix(i.kvreader, rv, storedRowPrefix) + + // now walk term keys in order and add them as well + if len(keys) > 0 { + it := i.kvreader.RangeIterator(keys[0], nil) + defer func() { + cerr := it.Close() + if cerr != nil { + rv <- cerr + } + }() + + for _, key := range keys { + it.Seek(key) + rkey, rval, valid := it.Current() + if !valid { + break + } + rck := make([]byte, len(rkey)) + copy(rck, key) + rcv := make([]byte, len(rval)) + copy(rcv, rval) + row, err := ParseFromKeyValue(rck, rcv) + if err != nil { + rv <- err + return + } + rv <- row + } + } + }() + + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go new file mode 100644 index 00000000..20d4eb34 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/field_dict.go @@ -0,0 +1,78 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/store" +) + +type UpsideDownCouchFieldDict struct { + indexReader *IndexReader + iterator store.KVIterator + dictRow *DictionaryRow + dictEntry *index.DictEntry + field uint16 +} + +func newUpsideDownCouchFieldDict(indexReader *IndexReader, field uint16, startTerm, endTerm []byte) (*UpsideDownCouchFieldDict, error) { + + startKey := NewDictionaryRow(startTerm, field, 0).Key() + if endTerm == nil { + endTerm = []byte{ByteSeparator} + } else { + endTerm = incrementBytes(endTerm) + } + endKey := NewDictionaryRow(endTerm, field, 0).Key() + + it := indexReader.kvreader.RangeIterator(startKey, endKey) + + return &UpsideDownCouchFieldDict{ + indexReader: indexReader, + iterator: it, + dictRow: &DictionaryRow{}, // Pre-alloced, reused row. + dictEntry: &index.DictEntry{}, // Pre-alloced, reused entry. + field: field, + }, nil + +} + +func (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) { + key, val, valid := r.iterator.Current() + if !valid { + return nil, nil + } + + err := r.dictRow.parseDictionaryK(key) + if err != nil { + return nil, fmt.Errorf("unexpected error parsing dictionary row key: %v", err) + } + err = r.dictRow.parseDictionaryV(val) + if err != nil { + return nil, fmt.Errorf("unexpected error parsing dictionary row val: %v", err) + } + r.dictEntry.Term = string(r.dictRow.term) + r.dictEntry.Count = r.dictRow.count + // advance the iterator to the next term + r.iterator.Next() + return r.dictEntry, nil + +} + +func (r *UpsideDownCouchFieldDict) Close() error { + return r.iterator.Close() +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go new file mode 100644 index 00000000..77d523c3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go @@ -0,0 +1,203 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/store" +) + +type IndexReader struct { + index *UpsideDownCouch + kvreader store.KVReader + docCount uint64 +} + +func (i *IndexReader) TermFieldReader(term []byte, fieldName string, includeFreq, includeNorm, includeTermVectors bool) (index.TermFieldReader, error) { + fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false) + if fieldExists { + return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex), includeFreq, includeNorm, includeTermVectors) + } + return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0), includeFreq, includeNorm, includeTermVectors) +} + +func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) { + return i.FieldDictRange(fieldName, nil, nil) +} + +func (i *IndexReader) FieldDictRange(fieldName string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { + fieldIndex, fieldExists := i.index.fieldCache.FieldNamed(fieldName, false) + if fieldExists { + return newUpsideDownCouchFieldDict(i, uint16(fieldIndex), startTerm, endTerm) + } + return newUpsideDownCouchFieldDict(i, ^uint16(0), []byte{ByteSeparator}, []byte{}) +} + +func (i *IndexReader) FieldDictPrefix(fieldName string, termPrefix []byte) (index.FieldDict, error) { + return i.FieldDictRange(fieldName, termPrefix, termPrefix) +} + +func (i *IndexReader) DocIDReaderAll() (index.DocIDReader, error) { + return newUpsideDownCouchDocIDReader(i) +} + +func (i *IndexReader) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + return newUpsideDownCouchDocIDReaderOnly(i, ids) +} + +func (i *IndexReader) Document(id string) (doc *document.Document, err error) { + // first hit the back index to confirm doc exists + var backIndexRow *BackIndexRow + backIndexRow, err = backIndexRowForDoc(i.kvreader, []byte(id)) + if err != nil { + return + } + if backIndexRow == nil { + return + } + doc = document.NewDocument(id) + storedRow := NewStoredRow([]byte(id), 0, []uint64{}, 'x', nil) + storedRowScanPrefix := storedRow.ScanPrefixForDoc() + it := i.kvreader.PrefixIterator(storedRowScanPrefix) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + key, val, valid := it.Current() + for valid { + safeVal := make([]byte, len(val)) + copy(safeVal, val) + var row *StoredRow + row, err = NewStoredRowKV(key, safeVal) + if err != nil { + doc = nil + return + } + if row != nil { + fieldName := i.index.fieldCache.FieldIndexed(row.field) + field := decodeFieldType(row.typ, fieldName, row.arrayPositions, row.value) + if field != nil { + doc.AddField(field) + } + } + + it.Next() + key, val, valid = it.Current() + } + return +} + +func (i *IndexReader) DocumentVisitFieldTerms(id index.IndexInternalID, fields []string, visitor index.DocumentFieldTermVisitor) error { + fieldsMap := make(map[uint16]string, len(fields)) + for _, f := range fields { + id, ok := i.index.fieldCache.FieldNamed(f, false) + if ok { + fieldsMap[id] = f + } + } + + tempRow := BackIndexRow{ + doc: id, + } + + keyBuf := GetRowBuffer() + if tempRow.KeySize() > len(keyBuf) { + keyBuf = make([]byte, 2*tempRow.KeySize()) + } + defer PutRowBuffer(keyBuf) + keySize, err := tempRow.KeyTo(keyBuf) + if err != nil { + return err + } + + value, err := i.kvreader.Get(keyBuf[:keySize]) + if err != nil { + return err + } + if value == nil { + return nil + } + + return visitBackIndexRow(value, func(field uint32, term []byte) { + if field, ok := fieldsMap[uint16(field)]; ok { + visitor(field, term) + } + }) +} + +func (i *IndexReader) Fields() (fields []string, err error) { + fields = make([]string, 0) + it := i.kvreader.PrefixIterator([]byte{'f'}) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + key, val, valid := it.Current() + for valid { + var row UpsideDownCouchRow + row, err = ParseFromKeyValue(key, val) + if err != nil { + fields = nil + return + } + if row != nil { + fieldRow, ok := row.(*FieldRow) + if ok { + fields = append(fields, fieldRow.name) + } + } + + it.Next() + key, val, valid = it.Current() + } + return +} + +func (i *IndexReader) GetInternal(key []byte) ([]byte, error) { + internalRow := NewInternalRow(key, nil) + return i.kvreader.Get(internalRow.Key()) +} + +func (i *IndexReader) DocCount() (uint64, error) { + return i.docCount, nil +} + +func (i *IndexReader) Close() error { + return i.kvreader.Close() +} + +func (i *IndexReader) ExternalID(id index.IndexInternalID) (string, error) { + return string(id), nil +} + +func (i *IndexReader) InternalID(id string) (index.IndexInternalID, error) { + return index.IndexInternalID(id), nil +} + +func incrementBytes(in []byte) []byte { + rv := make([]byte, len(in)) + copy(rv, in) + for i := len(rv) - 1; i >= 0; i-- { + rv[i] = rv[i] + 1 + if rv[i] != 0 { + // didn't overflow, so stop + break + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go new file mode 100644 index 00000000..76032bf5 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go @@ -0,0 +1,336 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "sort" + "sync/atomic" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/store" +) + +type UpsideDownCouchTermFieldReader struct { + count uint64 + indexReader *IndexReader + iterator store.KVIterator + term []byte + tfrNext *TermFrequencyRow + tfrPrealloc TermFrequencyRow + keyBuf []byte + field uint16 + includeTermVectors bool +} + +func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) { + bufNeeded := termFrequencyRowKeySize(term, nil) + if bufNeeded < dictionaryRowKeySize(term) { + bufNeeded = dictionaryRowKeySize(term) + } + buf := make([]byte, bufNeeded) + + bufUsed := dictionaryRowKeyTo(buf, field, term) + val, err := indexReader.kvreader.Get(buf[:bufUsed]) + if err != nil { + return nil, err + } + if val == nil { + atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) + rv := &UpsideDownCouchTermFieldReader{ + count: 0, + term: term, + field: field, + includeTermVectors: includeTermVectors, + } + rv.tfrNext = &rv.tfrPrealloc + return rv, nil + } + + count, err := dictionaryRowParseV(val) + if err != nil { + return nil, err + } + + bufUsed = termFrequencyRowKeyTo(buf, field, term, nil) + it := indexReader.kvreader.PrefixIterator(buf[:bufUsed]) + + atomic.AddUint64(&indexReader.index.stats.termSearchersStarted, uint64(1)) + return &UpsideDownCouchTermFieldReader{ + indexReader: indexReader, + iterator: it, + count: count, + term: term, + field: field, + includeTermVectors: includeTermVectors, + }, nil +} + +func (r *UpsideDownCouchTermFieldReader) Count() uint64 { + return r.count +} + +func (r *UpsideDownCouchTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + if r.iterator != nil { + // We treat tfrNext also like an initialization flag, which + // tells us whether we need to invoke the underlying + // iterator.Next(). The first time, don't call iterator.Next(). + if r.tfrNext != nil { + r.iterator.Next() + } else { + r.tfrNext = &r.tfrPrealloc + } + key, val, valid := r.iterator.Current() + if valid { + tfr := r.tfrNext + err := tfr.parseKDoc(key, r.term) + if err != nil { + return nil, err + } + err = tfr.parseV(val, r.includeTermVectors) + if err != nil { + return nil, err + } + rv := preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + rv.ID = append(rv.ID, tfr.doc...) + rv.Freq = tfr.freq + rv.Norm = float64(tfr.norm) + if tfr.vectors != nil { + rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) + } + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchTermFieldReader) Advance(docID index.IndexInternalID, preAlloced *index.TermFieldDoc) (rv *index.TermFieldDoc, err error) { + if r.iterator != nil { + if r.tfrNext == nil { + r.tfrNext = &TermFrequencyRow{} + } + tfr := InitTermFrequencyRow(r.tfrNext, r.term, r.field, docID, 0, 0) + r.keyBuf, err = tfr.KeyAppendTo(r.keyBuf[:0]) + if err != nil { + return nil, err + } + r.iterator.Seek(r.keyBuf) + key, val, valid := r.iterator.Current() + if valid { + err := tfr.parseKDoc(key, r.term) + if err != nil { + return nil, err + } + err = tfr.parseV(val, r.includeTermVectors) + if err != nil { + return nil, err + } + rv = preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + rv.ID = append(rv.ID, tfr.doc...) + rv.Freq = tfr.freq + rv.Norm = float64(tfr.norm) + if tfr.vectors != nil { + rv.Vectors = r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors) + } + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchTermFieldReader) Close() error { + if r.indexReader != nil { + atomic.AddUint64(&r.indexReader.index.stats.termSearchersFinished, uint64(1)) + } + if r.iterator != nil { + return r.iterator.Close() + } + return nil +} + +type UpsideDownCouchDocIDReader struct { + indexReader *IndexReader + iterator store.KVIterator + only []string + onlyPos int + onlyMode bool +} + +func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) { + + startBytes := []byte{0x0} + endBytes := []byte{0xff} + + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + }, nil +} + +func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) { + // ensure ids are sorted + sort.Strings(ids) + startBytes := []byte{0x0} + if len(ids) > 0 { + startBytes = []byte(ids[0]) + } + endBytes := []byte{0xff} + if len(ids) > 0 { + endBytes = incrementBytes([]byte(ids[len(ids)-1])) + } + bisr := NewBackIndexRow(startBytes, nil, nil) + bier := NewBackIndexRow(endBytes, nil, nil) + it := indexReader.kvreader.RangeIterator(bisr.Key(), bier.Key()) + + return &UpsideDownCouchDocIDReader{ + indexReader: indexReader, + iterator: it, + only: ids, + onlyMode: true, + }, nil +} + +func (r *UpsideDownCouchDocIDReader) Next() (index.IndexInternalID, error) { + key, val, valid := r.iterator.Current() + + if r.onlyMode { + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + ok := r.nextOnly() + if !ok { + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + + } else { + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Advance(docID index.IndexInternalID) (index.IndexInternalID, error) { + + if r.onlyMode { + r.onlyPos = sort.SearchStrings(r.only, string(docID)) + if r.onlyPos >= len(r.only) { + // advanced to key after our last only key + return nil, nil + } + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid := r.iterator.Current() + + var rv index.IndexInternalID + for valid && r.onlyPos < len(r.only) { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + if !bytes.Equal(br.doc, []byte(r.only[r.onlyPos])) { + // the only key we seek'd to didn't exist + // now look for the closest key that did exist in only + r.onlyPos = sort.SearchStrings(r.only, string(br.doc)) + if r.onlyPos >= len(r.only) { + // advanced to key after our last only key + return nil, nil + } + // now seek to this new only key + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + key, val, valid = r.iterator.Current() + continue + } else { + rv = append([]byte(nil), br.doc...) + break + } + } + if valid && r.onlyPos < len(r.only) { + ok := r.nextOnly() + if ok { + r.iterator.Seek(NewBackIndexRow([]byte(r.only[r.onlyPos]), nil, nil).Key()) + } + return rv, nil + } + } else { + bir := NewBackIndexRow(docID, nil, nil) + r.iterator.Seek(bir.Key()) + key, val, valid := r.iterator.Current() + if valid { + br, err := NewBackIndexRowKV(key, val) + if err != nil { + return nil, err + } + rv := append([]byte(nil), br.doc...) + r.iterator.Next() + return rv, nil + } + } + return nil, nil +} + +func (r *UpsideDownCouchDocIDReader) Close() error { + return r.iterator.Close() +} + +// move the r.only pos forward one, skipping duplicates +// return true if there is more data, or false if we got to the end of the list +func (r *UpsideDownCouchDocIDReader) nextOnly() bool { + + // advance 1 position, until we see a different key + // it's already sorted, so this skips duplicates + start := r.onlyPos + r.onlyPos++ + for r.onlyPos < len(r.only) && r.only[r.onlyPos] == r.only[start] { + start = r.onlyPos + r.onlyPos++ + } + // inidicate if we got to the end of the list + return r.onlyPos < len(r.only) +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go new file mode 100644 index 00000000..7e503ae0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go @@ -0,0 +1,1112 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + + "github.com/golang/protobuf/proto" +) + +const ByteSeparator byte = 0xff + +type UpsideDownCouchRowStream chan UpsideDownCouchRow + +type UpsideDownCouchRow interface { + KeySize() int + KeyTo([]byte) (int, error) + Key() []byte + Value() []byte + ValueSize() int + ValueTo([]byte) (int, error) +} + +func ParseFromKeyValue(key, value []byte) (UpsideDownCouchRow, error) { + if len(key) > 0 { + switch key[0] { + case 'v': + return NewVersionRowKV(key, value) + case 'f': + return NewFieldRowKV(key, value) + case 'd': + return NewDictionaryRowKV(key, value) + case 't': + return NewTermFrequencyRowKV(key, value) + case 'b': + return NewBackIndexRowKV(key, value) + case 's': + return NewStoredRowKV(key, value) + case 'i': + return NewInternalRowKV(key, value) + } + return nil, fmt.Errorf("Unknown field type '%s'", string(key[0])) + } + return nil, fmt.Errorf("Invalid empty key") +} + +// VERSION + +type VersionRow struct { + version uint8 +} + +func (v *VersionRow) Key() []byte { + return []byte{'v'} +} + +func (v *VersionRow) KeySize() int { + return 1 +} + +func (v *VersionRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'v' + return 1, nil +} + +func (v *VersionRow) Value() []byte { + return []byte{byte(v.version)} +} + +func (v *VersionRow) ValueSize() int { + return 1 +} + +func (v *VersionRow) ValueTo(buf []byte) (int, error) { + buf[0] = v.version + return 1, nil +} + +func (v *VersionRow) String() string { + return fmt.Sprintf("Version: %d", v.version) +} + +func NewVersionRow(version uint8) *VersionRow { + return &VersionRow{ + version: version, + } +} + +func NewVersionRowKV(key, value []byte) (*VersionRow, error) { + rv := VersionRow{} + buf := bytes.NewBuffer(value) + err := binary.Read(buf, binary.LittleEndian, &rv.version) + if err != nil { + return nil, err + } + return &rv, nil +} + +// INTERNAL STORAGE + +type InternalRow struct { + key []byte + val []byte +} + +func (i *InternalRow) Key() []byte { + buf := make([]byte, i.KeySize()) + size, _ := i.KeyTo(buf) + return buf[:size] +} + +func (i *InternalRow) KeySize() int { + return len(i.key) + 1 +} + +func (i *InternalRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'i' + actual := copy(buf[1:], i.key) + return 1 + actual, nil +} + +func (i *InternalRow) Value() []byte { + return i.val +} + +func (i *InternalRow) ValueSize() int { + return len(i.val) +} + +func (i *InternalRow) ValueTo(buf []byte) (int, error) { + actual := copy(buf, i.val) + return actual, nil +} + +func (i *InternalRow) String() string { + return fmt.Sprintf("InternalStore - Key: %s (% x) Val: %s (% x)", i.key, i.key, i.val, i.val) +} + +func NewInternalRow(key, val []byte) *InternalRow { + return &InternalRow{ + key: key, + val: val, + } +} + +func NewInternalRowKV(key, value []byte) (*InternalRow, error) { + rv := InternalRow{} + rv.key = key[1:] + rv.val = value + return &rv, nil +} + +// FIELD definition + +type FieldRow struct { + index uint16 + name string +} + +func (f *FieldRow) Key() []byte { + buf := make([]byte, f.KeySize()) + size, _ := f.KeyTo(buf) + return buf[:size] +} + +func (f *FieldRow) KeySize() int { + return 3 +} + +func (f *FieldRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'f' + binary.LittleEndian.PutUint16(buf[1:3], f.index) + return 3, nil +} + +func (f *FieldRow) Value() []byte { + return append([]byte(f.name), ByteSeparator) +} + +func (f *FieldRow) ValueSize() int { + return len(f.name) + 1 +} + +func (f *FieldRow) ValueTo(buf []byte) (int, error) { + size := copy(buf, f.name) + buf[size] = ByteSeparator + return size + 1, nil +} + +func (f *FieldRow) String() string { + return fmt.Sprintf("Field: %d Name: %s", f.index, f.name) +} + +func NewFieldRow(index uint16, name string) *FieldRow { + return &FieldRow{ + index: index, + name: name, + } +} + +func NewFieldRowKV(key, value []byte) (*FieldRow, error) { + rv := FieldRow{} + + buf := bytes.NewBuffer(key) + _, err := buf.ReadByte() // type + if err != nil { + return nil, err + } + err = binary.Read(buf, binary.LittleEndian, &rv.index) + if err != nil { + return nil, err + } + + buf = bytes.NewBuffer(value) + rv.name, err = buf.ReadString(ByteSeparator) + if err != nil { + return nil, err + } + rv.name = rv.name[:len(rv.name)-1] // trim off separator byte + + return &rv, nil +} + +// DICTIONARY + +const DictionaryRowMaxValueSize = binary.MaxVarintLen64 + +type DictionaryRow struct { + term []byte + count uint64 + field uint16 +} + +func (dr *DictionaryRow) Key() []byte { + buf := make([]byte, dr.KeySize()) + size, _ := dr.KeyTo(buf) + return buf[:size] +} + +func (dr *DictionaryRow) KeySize() int { + return dictionaryRowKeySize(dr.term) +} + +func dictionaryRowKeySize(term []byte) int { + return len(term) + 3 +} + +func (dr *DictionaryRow) KeyTo(buf []byte) (int, error) { + return dictionaryRowKeyTo(buf, dr.field, dr.term), nil +} + +func dictionaryRowKeyTo(buf []byte, field uint16, term []byte) int { + buf[0] = 'd' + binary.LittleEndian.PutUint16(buf[1:3], field) + size := copy(buf[3:], term) + return size + 3 +} + +func (dr *DictionaryRow) Value() []byte { + buf := make([]byte, dr.ValueSize()) + size, _ := dr.ValueTo(buf) + return buf[:size] +} + +func (dr *DictionaryRow) ValueSize() int { + return DictionaryRowMaxValueSize +} + +func (dr *DictionaryRow) ValueTo(buf []byte) (int, error) { + used := binary.PutUvarint(buf, dr.count) + return used, nil +} + +func (dr *DictionaryRow) String() string { + return fmt.Sprintf("Dictionary Term: `%s` Field: %d Count: %d ", string(dr.term), dr.field, dr.count) +} + +func NewDictionaryRow(term []byte, field uint16, count uint64) *DictionaryRow { + return &DictionaryRow{ + term: term, + field: field, + count: count, + } +} + +func NewDictionaryRowKV(key, value []byte) (*DictionaryRow, error) { + rv, err := NewDictionaryRowK(key) + if err != nil { + return nil, err + } + + err = rv.parseDictionaryV(value) + if err != nil { + return nil, err + } + return rv, nil + +} + +func NewDictionaryRowK(key []byte) (*DictionaryRow, error) { + rv := &DictionaryRow{} + err := rv.parseDictionaryK(key) + if err != nil { + return nil, err + } + return rv, nil +} + +func (dr *DictionaryRow) parseDictionaryK(key []byte) error { + dr.field = binary.LittleEndian.Uint16(key[1:3]) + if dr.term != nil { + dr.term = dr.term[:0] + } + dr.term = append(dr.term, key[3:]...) + return nil +} + +func (dr *DictionaryRow) parseDictionaryV(value []byte) error { + count, err := dictionaryRowParseV(value) + if err != nil { + return err + } + dr.count = count + return nil +} + +func dictionaryRowParseV(value []byte) (uint64, error) { + count, nread := binary.Uvarint(value) + if nread <= 0 { + return 0, fmt.Errorf("DictionaryRow parse Uvarint error, nread: %d", nread) + } + return count, nil +} + +// TERM FIELD FREQUENCY + +type TermVector struct { + field uint16 + arrayPositions []uint64 + pos uint64 + start uint64 + end uint64 +} + +func (tv *TermVector) String() string { + return fmt.Sprintf("Field: %d Pos: %d Start: %d End %d ArrayPositions: %#v", tv.field, tv.pos, tv.start, tv.end, tv.arrayPositions) +} + +type TermFrequencyRow struct { + term []byte + doc []byte + freq uint64 + vectors []*TermVector + norm float32 + field uint16 +} + +func (tfr *TermFrequencyRow) Term() []byte { + return tfr.term +} + +func (tfr *TermFrequencyRow) Freq() uint64 { + return tfr.freq +} + +func (tfr *TermFrequencyRow) ScanPrefixForField() []byte { + buf := make([]byte, 3) + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], tfr.field) + return buf +} + +func (tfr *TermFrequencyRow) ScanPrefixForFieldTermPrefix() []byte { + buf := make([]byte, 3+len(tfr.term)) + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], tfr.field) + copy(buf[3:], tfr.term) + return buf +} + +func (tfr *TermFrequencyRow) ScanPrefixForFieldTerm() []byte { + buf := make([]byte, 3+len(tfr.term)+1) + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], tfr.field) + termLen := copy(buf[3:], tfr.term) + buf[3+termLen] = ByteSeparator + return buf +} + +func (tfr *TermFrequencyRow) Key() []byte { + buf := make([]byte, tfr.KeySize()) + size, _ := tfr.KeyTo(buf) + return buf[:size] +} + +func (tfr *TermFrequencyRow) KeySize() int { + return termFrequencyRowKeySize(tfr.term, tfr.doc) +} + +func termFrequencyRowKeySize(term, doc []byte) int { + return 3 + len(term) + 1 + len(doc) +} + +func (tfr *TermFrequencyRow) KeyTo(buf []byte) (int, error) { + return termFrequencyRowKeyTo(buf, tfr.field, tfr.term, tfr.doc), nil +} + +func termFrequencyRowKeyTo(buf []byte, field uint16, term, doc []byte) int { + buf[0] = 't' + binary.LittleEndian.PutUint16(buf[1:3], field) + termLen := copy(buf[3:], term) + buf[3+termLen] = ByteSeparator + docLen := copy(buf[3+termLen+1:], doc) + return 3 + termLen + 1 + docLen +} + +func (tfr *TermFrequencyRow) KeyAppendTo(buf []byte) ([]byte, error) { + keySize := tfr.KeySize() + if cap(buf) < keySize { + buf = make([]byte, keySize) + } + actualSize, err := tfr.KeyTo(buf[0:keySize]) + return buf[0:actualSize], err +} + +func (tfr *TermFrequencyRow) DictionaryRowKey() []byte { + dr := NewDictionaryRow(tfr.term, tfr.field, 0) + return dr.Key() +} + +func (tfr *TermFrequencyRow) DictionaryRowKeySize() int { + dr := NewDictionaryRow(tfr.term, tfr.field, 0) + return dr.KeySize() +} + +func (tfr *TermFrequencyRow) DictionaryRowKeyTo(buf []byte) (int, error) { + dr := NewDictionaryRow(tfr.term, tfr.field, 0) + return dr.KeyTo(buf) +} + +func (tfr *TermFrequencyRow) Value() []byte { + buf := make([]byte, tfr.ValueSize()) + size, _ := tfr.ValueTo(buf) + return buf[:size] +} + +func (tfr *TermFrequencyRow) ValueSize() int { + bufLen := binary.MaxVarintLen64 + binary.MaxVarintLen64 + for _, vector := range tfr.vectors { + bufLen += (binary.MaxVarintLen64 * 4) + (1+len(vector.arrayPositions))*binary.MaxVarintLen64 + } + return bufLen +} + +func (tfr *TermFrequencyRow) ValueTo(buf []byte) (int, error) { + used := binary.PutUvarint(buf[:binary.MaxVarintLen64], tfr.freq) + + normuint32 := math.Float32bits(tfr.norm) + newbuf := buf[used : used+binary.MaxVarintLen64] + used += binary.PutUvarint(newbuf, uint64(normuint32)) + + for _, vector := range tfr.vectors { + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], uint64(vector.field)) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.pos) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.start) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], vector.end) + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], uint64(len(vector.arrayPositions))) + for _, arrayPosition := range vector.arrayPositions { + used += binary.PutUvarint(buf[used:used+binary.MaxVarintLen64], arrayPosition) + } + } + return used, nil +} + +func (tfr *TermFrequencyRow) String() string { + return fmt.Sprintf("Term: `%s` Field: %d DocId: `%s` Frequency: %d Norm: %f Vectors: %v", string(tfr.term), tfr.field, string(tfr.doc), tfr.freq, tfr.norm, tfr.vectors) +} + +func InitTermFrequencyRow(tfr *TermFrequencyRow, term []byte, field uint16, docID []byte, freq uint64, norm float32) *TermFrequencyRow { + tfr.term = term + tfr.field = field + tfr.doc = docID + tfr.freq = freq + tfr.norm = norm + return tfr +} + +func NewTermFrequencyRow(term []byte, field uint16, docID []byte, freq uint64, norm float32) *TermFrequencyRow { + return &TermFrequencyRow{ + term: term, + field: field, + doc: docID, + freq: freq, + norm: norm, + } +} + +func NewTermFrequencyRowWithTermVectors(term []byte, field uint16, docID []byte, freq uint64, norm float32, vectors []*TermVector) *TermFrequencyRow { + return &TermFrequencyRow{ + term: term, + field: field, + doc: docID, + freq: freq, + norm: norm, + vectors: vectors, + } +} + +func NewTermFrequencyRowK(key []byte) (*TermFrequencyRow, error) { + rv := &TermFrequencyRow{} + err := rv.parseK(key) + if err != nil { + return nil, err + } + return rv, nil +} + +func (tfr *TermFrequencyRow) parseK(key []byte) error { + keyLen := len(key) + if keyLen < 3 { + return fmt.Errorf("invalid term frequency key, no valid field") + } + tfr.field = binary.LittleEndian.Uint16(key[1:3]) + + termEndPos := bytes.IndexByte(key[3:], ByteSeparator) + if termEndPos < 0 { + return fmt.Errorf("invalid term frequency key, no byte separator terminating term") + } + tfr.term = key[3 : 3+termEndPos] + + docLen := keyLen - (3 + termEndPos + 1) + if docLen < 1 { + return fmt.Errorf("invalid term frequency key, empty docid") + } + tfr.doc = key[3+termEndPos+1:] + + return nil +} + +func (tfr *TermFrequencyRow) parseKDoc(key []byte, term []byte) error { + tfr.doc = key[3+len(term)+1:] + if len(tfr.doc) <= 0 { + return fmt.Errorf("invalid term frequency key, empty docid") + } + + return nil +} + +func (tfr *TermFrequencyRow) parseV(value []byte, includeTermVectors bool) error { + var bytesRead int + tfr.freq, bytesRead = binary.Uvarint(value) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, invalid frequency") + } + currOffset := bytesRead + + var norm uint64 + norm, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, no norm") + } + currOffset += bytesRead + + tfr.norm = math.Float32frombits(uint32(norm)) + + tfr.vectors = nil + if !includeTermVectors { + return nil + } + + var field uint64 + field, bytesRead = binary.Uvarint(value[currOffset:]) + for bytesRead > 0 { + currOffset += bytesRead + tv := TermVector{} + tv.field = uint16(field) + // at this point we expect at least one term vector + if tfr.vectors == nil { + tfr.vectors = make([]*TermVector, 0) + } + + tv.pos, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no position") + } + currOffset += bytesRead + + tv.start, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no start") + } + currOffset += bytesRead + + tv.end, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no end") + } + currOffset += bytesRead + + var arrayPositionsLen uint64 = 0 + arrayPositionsLen, bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no arrayPositionLen") + } + currOffset += bytesRead + + if arrayPositionsLen > 0 { + tv.arrayPositions = make([]uint64, arrayPositionsLen) + for i := 0; uint64(i) < arrayPositionsLen; i++ { + tv.arrayPositions[i], bytesRead = binary.Uvarint(value[currOffset:]) + if bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector contains no arrayPosition of index %d", i) + } + currOffset += bytesRead + } + } + + tfr.vectors = append(tfr.vectors, &tv) + // try to read next record (may not exist) + field, bytesRead = binary.Uvarint(value[currOffset:]) + } + if len(value[currOffset:]) > 0 && bytesRead <= 0 { + return fmt.Errorf("invalid term frequency value, vector field invalid") + } + + return nil +} + +func NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) { + rv, err := NewTermFrequencyRowK(key) + if err != nil { + return nil, err + } + + err = rv.parseV(value, true) + if err != nil { + return nil, err + } + return rv, nil + +} + +type BackIndexRow struct { + doc []byte + termsEntries []*BackIndexTermsEntry + storedEntries []*BackIndexStoreEntry +} + +func (br *BackIndexRow) AllTermKeys() [][]byte { + if br == nil { + return nil + } + rv := make([][]byte, 0, len(br.termsEntries)) // FIXME this underestimates severely + for _, termsEntry := range br.termsEntries { + for i := range termsEntry.Terms { + termRow := NewTermFrequencyRow([]byte(termsEntry.Terms[i]), uint16(termsEntry.GetField()), br.doc, 0, 0) + rv = append(rv, termRow.Key()) + } + } + return rv +} + +func (br *BackIndexRow) AllStoredKeys() [][]byte { + if br == nil { + return nil + } + rv := make([][]byte, len(br.storedEntries)) + for i, storedEntry := range br.storedEntries { + storedRow := NewStoredRow(br.doc, uint16(storedEntry.GetField()), storedEntry.GetArrayPositions(), 'x', []byte{}) + rv[i] = storedRow.Key() + } + return rv +} + +func (br *BackIndexRow) Key() []byte { + buf := make([]byte, br.KeySize()) + size, _ := br.KeyTo(buf) + return buf[:size] +} + +func (br *BackIndexRow) KeySize() int { + return len(br.doc) + 1 +} + +func (br *BackIndexRow) KeyTo(buf []byte) (int, error) { + buf[0] = 'b' + used := copy(buf[1:], br.doc) + return used + 1, nil +} + +func (br *BackIndexRow) Value() []byte { + buf := make([]byte, br.ValueSize()) + size, _ := br.ValueTo(buf) + return buf[:size] +} + +func (br *BackIndexRow) ValueSize() int { + birv := &BackIndexRowValue{ + TermsEntries: br.termsEntries, + StoredEntries: br.storedEntries, + } + return birv.Size() +} + +func (br *BackIndexRow) ValueTo(buf []byte) (int, error) { + birv := &BackIndexRowValue{ + TermsEntries: br.termsEntries, + StoredEntries: br.storedEntries, + } + return birv.MarshalTo(buf) +} + +func (br *BackIndexRow) String() string { + return fmt.Sprintf("Backindex DocId: `%s` Terms Entries: %v, Stored Entries: %v", string(br.doc), br.termsEntries, br.storedEntries) +} + +func NewBackIndexRow(docID []byte, entries []*BackIndexTermsEntry, storedFields []*BackIndexStoreEntry) *BackIndexRow { + return &BackIndexRow{ + doc: docID, + termsEntries: entries, + storedEntries: storedFields, + } +} + +func NewBackIndexRowKV(key, value []byte) (*BackIndexRow, error) { + rv := BackIndexRow{} + + buf := bytes.NewBuffer(key) + _, err := buf.ReadByte() // type + if err != nil { + return nil, err + } + + rv.doc, err = buf.ReadBytes(ByteSeparator) + if err == io.EOF && len(rv.doc) < 1 { + err = fmt.Errorf("invalid doc length 0 - % x", key) + } + if err != nil && err != io.EOF { + return nil, err + } else if err == nil { + rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte + } + + var birv BackIndexRowValue + err = proto.Unmarshal(value, &birv) + if err != nil { + return nil, err + } + rv.termsEntries = birv.TermsEntries + rv.storedEntries = birv.StoredEntries + + return &rv, nil +} + +// STORED + +type StoredRow struct { + doc []byte + field uint16 + arrayPositions []uint64 + typ byte + value []byte +} + +func (s *StoredRow) Key() []byte { + buf := make([]byte, s.KeySize()) + size, _ := s.KeyTo(buf) + return buf[0:size] +} + +func (s *StoredRow) KeySize() int { + return 1 + len(s.doc) + 1 + 2 + (binary.MaxVarintLen64 * len(s.arrayPositions)) +} + +func (s *StoredRow) KeyTo(buf []byte) (int, error) { + docLen := len(s.doc) + buf[0] = 's' + copy(buf[1:], s.doc) + buf[1+docLen] = ByteSeparator + binary.LittleEndian.PutUint16(buf[1+docLen+1:], s.field) + bytesUsed := 1 + docLen + 1 + 2 + for _, arrayPosition := range s.arrayPositions { + varbytes := binary.PutUvarint(buf[bytesUsed:], arrayPosition) + bytesUsed += varbytes + } + return bytesUsed, nil +} + +func (s *StoredRow) Value() []byte { + buf := make([]byte, s.ValueSize()) + size, _ := s.ValueTo(buf) + return buf[:size] +} + +func (s *StoredRow) ValueSize() int { + return len(s.value) + 1 +} + +func (s *StoredRow) ValueTo(buf []byte) (int, error) { + buf[0] = s.typ + used := copy(buf[1:], s.value) + return used + 1, nil +} + +func (s *StoredRow) String() string { + return fmt.Sprintf("Document: %s Field %d, Array Positions: %v, Type: %s Value: %s", s.doc, s.field, s.arrayPositions, string(s.typ), s.value) +} + +func (s *StoredRow) ScanPrefixForDoc() []byte { + docLen := len(s.doc) + buf := make([]byte, 1+docLen+1) + buf[0] = 's' + copy(buf[1:], s.doc) + buf[1+docLen] = ByteSeparator + return buf +} + +func NewStoredRow(docID []byte, field uint16, arrayPositions []uint64, typ byte, value []byte) *StoredRow { + return &StoredRow{ + doc: docID, + field: field, + arrayPositions: arrayPositions, + typ: typ, + value: value, + } +} + +func NewStoredRowK(key []byte) (*StoredRow, error) { + rv := StoredRow{} + + buf := bytes.NewBuffer(key) + _, err := buf.ReadByte() // type + if err != nil { + return nil, err + } + + rv.doc, err = buf.ReadBytes(ByteSeparator) + if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator + err = fmt.Errorf("invalid doc length 0") + return nil, err + } + + rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte + + err = binary.Read(buf, binary.LittleEndian, &rv.field) + if err != nil { + return nil, err + } + + rv.arrayPositions = make([]uint64, 0) + nextArrayPos, err := binary.ReadUvarint(buf) + for err == nil { + rv.arrayPositions = append(rv.arrayPositions, nextArrayPos) + nextArrayPos, err = binary.ReadUvarint(buf) + } + return &rv, nil +} + +func NewStoredRowKV(key, value []byte) (*StoredRow, error) { + rv, err := NewStoredRowK(key) + if err != nil { + return nil, err + } + rv.typ = value[0] + rv.value = value[1:] + return rv, nil +} + +type backIndexFieldTermVisitor func(field uint32, term []byte) + +// visitBackIndexRow is designed to process a protobuf encoded +// value, without creating unnecessary garbage. Instead values are passed +// to a callback, inspected first, and only copied if necessary. +// Due to the fact that this borrows from generated code, it must be marnually +// updated if the protobuf definition changes. +// +// This code originates from: +// func (m *BackIndexRowValue) Unmarshal(data []byte) error +// the sections which create garbage or parse unintersting sections +// have been commented out. This was done by design to allow for easier +// merging in the future if that original function is regenerated +func visitBackIndexRow(data []byte, callback backIndexFieldTermVisitor) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TermsEntries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + msglen + if msglen < 0 { + return ErrInvalidLengthUpsidedown + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + // dont parse term entries + // m.TermsEntries = append(m.TermsEntries, &BackIndexTermsEntry{}) + // if err := m.TermsEntries[len(m.TermsEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + // return err + // } + // instead, inspect them + if err := visitBackIndexRowFieldTerms(data[iNdEx:postIndex], callback); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StoredEntries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + msglen + if msglen < 0 { + return ErrInvalidLengthUpsidedown + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + // don't parse stored entries + // m.StoredEntries = append(m.StoredEntries, &BackIndexStoreEntry{}) + // if err := m.StoredEntries[len(m.StoredEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + // return err + // } + iNdEx = postIndex + default: + var sizeOfWire int + for { + sizeOfWire++ + wire >>= 7 + if wire == 0 { + break + } + } + iNdEx -= sizeOfWire + skippy, err := skipUpsidedown(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthUpsidedown + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + // don't track unrecognized data + //m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + return nil +} + +// visitBackIndexRowFieldTerms is designed to process a protobuf encoded +// sub-value within the BackIndexRowValue, without creating unnecessary garbage. +// Instead values are passed to a callback, inspected first, and only copied if +// necessary. Due to the fact that this borrows from generated code, it must +// be marnually updated if the protobuf definition changes. +// +// This code originates from: +// func (m *BackIndexTermsEntry) Unmarshal(data []byte) error { +// the sections which create garbage or parse uninteresting sections +// have been commented out. This was done by design to allow for easier +// merging in the future if that original function is regenerated +func visitBackIndexRowFieldTerms(data []byte, callback backIndexFieldTermVisitor) error { + var theField uint32 + + var hasFields [1]uint64 + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + // m.Field = &v + theField = v + hasFields[0] |= uint64(0x00000001) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Terms", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + int(stringLen) + if postIndex > l { + return io.ErrUnexpectedEOF + } + //m.Terms = append(m.Terms, string(data[iNdEx:postIndex])) + callback(theField, data[iNdEx:postIndex]) + iNdEx = postIndex + default: + var sizeOfWire int + for { + sizeOfWire++ + wire >>= 7 + if wire == 0 { + break + } + } + iNdEx -= sizeOfWire + skippy, err := skipUpsidedown(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthUpsidedown + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + //m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + // if hasFields[0]&uint64(0x00000001) == 0 { + // return new(github_com_golang_protobuf_proto.RequiredNotSetError) + // } + + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/row_merge.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go similarity index 68% rename from vendor/github.com/blevesearch/bleve/index/upside_down/row_merge.go rename to vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go index fb9d987a..39172ade 100644 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/row_merge.go +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/row_merge.go @@ -1,13 +1,18 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package upside_down +package upsidedown import ( "encoding/binary" @@ -61,8 +66,9 @@ func (m *upsideDownMerge) FullMerge(key, existingValue []byte, operands [][]byte func (m *upsideDownMerge) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) { left := int64(binary.LittleEndian.Uint64(leftOperand)) right := int64(binary.LittleEndian.Uint64(rightOperand)) - binary.LittleEndian.PutUint64(leftOperand, uint64(left+right)) - return leftOperand, true + rv := make([]byte, 8) + binary.LittleEndian.PutUint64(rv, uint64(left+right)) + return rv, true } func (m *upsideDownMerge) Name() string { diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go new file mode 100644 index 00000000..a148ab70 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/stats.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package upsidedown + +import ( + "encoding/json" + "sync/atomic" + + "github.com/blevesearch/bleve/index/store" +) + +type indexStat struct { + updates, deletes, batches, errors uint64 + analysisTime, indexTime uint64 + termSearchersStarted uint64 + termSearchersFinished uint64 + numPlainTextBytesIndexed uint64 + i *UpsideDownCouch +} + +func (i *indexStat) statsMap() map[string]interface{} { + m := map[string]interface{}{} + m["updates"] = atomic.LoadUint64(&i.updates) + m["deletes"] = atomic.LoadUint64(&i.deletes) + m["batches"] = atomic.LoadUint64(&i.batches) + m["errors"] = atomic.LoadUint64(&i.errors) + m["analysis_time"] = atomic.LoadUint64(&i.analysisTime) + m["index_time"] = atomic.LoadUint64(&i.indexTime) + m["term_searchers_started"] = atomic.LoadUint64(&i.termSearchersStarted) + m["term_searchers_finished"] = atomic.LoadUint64(&i.termSearchersFinished) + m["num_plain_text_bytes_indexed"] = atomic.LoadUint64(&i.numPlainTextBytesIndexed) + + if o, ok := i.i.store.(store.KVStoreStats); ok { + m["kv"] = o.StatsMap() + } + + return m +} + +func (i *indexStat) MarshalJSON() ([]byte, error) { + m := i.statsMap() + return json.Marshal(m) +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go new file mode 100644 index 00000000..a8ef538e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go @@ -0,0 +1,1072 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate protoc --gofast_out=. upsidedown.proto + +package upsidedown + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "math" + "sync" + "sync/atomic" + "time" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/registry" + + "github.com/golang/protobuf/proto" +) + +const Name = "upside_down" + +// RowBufferSize should ideally this is sized to be the smallest +// size that can contain an index row key and its corresponding +// value. It is not a limit, if need be a larger buffer is +// allocated, but performance will be more optimal if *most* +// rows fit this size. +const RowBufferSize = 4 * 1024 + +var VersionKey = []byte{'v'} + +const Version uint8 = 7 + +var IncompatibleVersion = fmt.Errorf("incompatible version, %d is supported", Version) + +type UpsideDownCouch struct { + version uint8 + path string + storeName string + storeConfig map[string]interface{} + store store.KVStore + fieldCache *index.FieldCache + analysisQueue *index.AnalysisQueue + stats *indexStat + + m sync.RWMutex + // fields protected by m + docCount uint64 + + writeMutex sync.Mutex +} + +type docBackIndexRow struct { + docID string + doc *document.Document // If deletion, doc will be nil. + backIndexRow *BackIndexRow +} + +func NewUpsideDownCouch(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) { + rv := &UpsideDownCouch{ + version: Version, + fieldCache: index.NewFieldCache(), + storeName: storeName, + storeConfig: storeConfig, + analysisQueue: analysisQueue, + } + rv.stats = &indexStat{i: rv} + return rv, nil +} + +func (udc *UpsideDownCouch) init(kvwriter store.KVWriter) (err error) { + // version marker + rowsAll := [][]UpsideDownCouchRow{ + {NewVersionRow(udc.version)}, + } + + err = udc.batchRows(kvwriter, nil, rowsAll, nil) + return +} + +func (udc *UpsideDownCouch) loadSchema(kvreader store.KVReader) (err error) { + + it := kvreader.PrefixIterator([]byte{'f'}) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + key, val, valid := it.Current() + for valid { + var fieldRow *FieldRow + fieldRow, err = NewFieldRowKV(key, val) + if err != nil { + return + } + udc.fieldCache.AddExisting(fieldRow.name, fieldRow.index) + + it.Next() + key, val, valid = it.Current() + } + + val, err = kvreader.Get([]byte{'v'}) + if err != nil { + return + } + var vr *VersionRow + vr, err = NewVersionRowKV([]byte{'v'}, val) + if err != nil { + return + } + if vr.version != Version { + err = IncompatibleVersion + return + } + + return +} + +var rowBufferPool sync.Pool + +func GetRowBuffer() []byte { + if rb, ok := rowBufferPool.Get().([]byte); ok { + return rb + } else { + return make([]byte, RowBufferSize) + } +} + +func PutRowBuffer(buf []byte) { + rowBufferPool.Put(buf) +} + +func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]UpsideDownCouchRow, updateRowsAll [][]UpsideDownCouchRow, deleteRowsAll [][]UpsideDownCouchRow) (err error) { + dictionaryDeltas := make(map[string]int64) + + // count up bytes needed for buffering. + addNum := 0 + addKeyBytes := 0 + addValBytes := 0 + + updateNum := 0 + updateKeyBytes := 0 + updateValBytes := 0 + + deleteNum := 0 + deleteKeyBytes := 0 + + rowBuf := GetRowBuffer() + + for _, addRows := range addRowsAll { + for _, row := range addRows { + tfr, ok := row.(*TermFrequencyRow) + if ok { + if tfr.DictionaryRowKeySize() > len(rowBuf) { + rowBuf = make([]byte, tfr.DictionaryRowKeySize()) + } + dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf) + if err != nil { + return err + } + dictionaryDeltas[string(rowBuf[:dictKeySize])] += 1 + } + addKeyBytes += row.KeySize() + addValBytes += row.ValueSize() + } + addNum += len(addRows) + } + + for _, updateRows := range updateRowsAll { + for _, row := range updateRows { + updateKeyBytes += row.KeySize() + updateValBytes += row.ValueSize() + } + updateNum += len(updateRows) + } + + for _, deleteRows := range deleteRowsAll { + for _, row := range deleteRows { + tfr, ok := row.(*TermFrequencyRow) + if ok { + // need to decrement counter + if tfr.DictionaryRowKeySize() > len(rowBuf) { + rowBuf = make([]byte, tfr.DictionaryRowKeySize()) + } + dictKeySize, err := tfr.DictionaryRowKeyTo(rowBuf) + if err != nil { + return err + } + dictionaryDeltas[string(rowBuf[:dictKeySize])] -= 1 + } + deleteKeyBytes += row.KeySize() + } + deleteNum += len(deleteRows) + } + + PutRowBuffer(rowBuf) + + mergeNum := len(dictionaryDeltas) + mergeKeyBytes := 0 + mergeValBytes := mergeNum * DictionaryRowMaxValueSize + + for dictRowKey := range dictionaryDeltas { + mergeKeyBytes += len(dictRowKey) + } + + // prepare batch + totBytes := addKeyBytes + addValBytes + + updateKeyBytes + updateValBytes + + deleteKeyBytes + + 2*(mergeKeyBytes+mergeValBytes) + + buf, wb, err := writer.NewBatchEx(store.KVBatchOptions{ + TotalBytes: totBytes, + NumSets: addNum + updateNum, + NumDeletes: deleteNum, + NumMerges: mergeNum, + }) + if err != nil { + return err + } + defer func() { + _ = wb.Close() + }() + + // fill the batch + for _, addRows := range addRowsAll { + for _, row := range addRows { + keySize, err := row.KeyTo(buf) + if err != nil { + return err + } + valSize, err := row.ValueTo(buf[keySize:]) + if err != nil { + return err + } + wb.Set(buf[:keySize], buf[keySize:keySize+valSize]) + buf = buf[keySize+valSize:] + } + } + + for _, updateRows := range updateRowsAll { + for _, row := range updateRows { + keySize, err := row.KeyTo(buf) + if err != nil { + return err + } + valSize, err := row.ValueTo(buf[keySize:]) + if err != nil { + return err + } + wb.Set(buf[:keySize], buf[keySize:keySize+valSize]) + buf = buf[keySize+valSize:] + } + } + + for _, deleteRows := range deleteRowsAll { + for _, row := range deleteRows { + keySize, err := row.KeyTo(buf) + if err != nil { + return err + } + wb.Delete(buf[:keySize]) + buf = buf[keySize:] + } + } + + for dictRowKey, delta := range dictionaryDeltas { + dictRowKeyLen := copy(buf, dictRowKey) + binary.LittleEndian.PutUint64(buf[dictRowKeyLen:], uint64(delta)) + wb.Merge(buf[:dictRowKeyLen], buf[dictRowKeyLen:dictRowKeyLen+DictionaryRowMaxValueSize]) + buf = buf[dictRowKeyLen+DictionaryRowMaxValueSize:] + } + + // write out the batch + return writer.ExecuteBatch(wb) +} + +func (udc *UpsideDownCouch) Open() (err error) { + //acquire the write mutex for the duratin of Open() + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + // open the kv store + storeConstructor := registry.KVStoreConstructorByName(udc.storeName) + if storeConstructor == nil { + err = index.ErrorUnknownStorageType + return + } + + // now open the store + udc.store, err = storeConstructor(&mergeOperator, udc.storeConfig) + if err != nil { + return + } + + // start a reader to look at the index + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + return + } + + var value []byte + value, err = kvreader.Get(VersionKey) + if err != nil { + _ = kvreader.Close() + return + } + + if value != nil { + err = udc.loadSchema(kvreader) + if err != nil { + _ = kvreader.Close() + return + } + + // set doc count + udc.m.Lock() + udc.docCount, err = udc.countDocs(kvreader) + udc.m.Unlock() + + err = kvreader.Close() + } else { + // new index, close the reader and open writer to init + err = kvreader.Close() + if err != nil { + return + } + + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := kvwriter.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + // init the index + err = udc.init(kvwriter) + } + + return +} + +func (udc *UpsideDownCouch) countDocs(kvreader store.KVReader) (count uint64, err error) { + it := kvreader.PrefixIterator([]byte{'b'}) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + _, _, valid := it.Current() + for valid { + count++ + it.Next() + _, _, valid = it.Current() + } + + return +} + +func (udc *UpsideDownCouch) rowCount() (count uint64, err error) { + // start an isolated reader for use during the rowcount + kvreader, err := udc.store.Reader() + if err != nil { + return + } + defer func() { + if cerr := kvreader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + it := kvreader.RangeIterator(nil, nil) + defer func() { + if cerr := it.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + _, _, valid := it.Current() + for valid { + count++ + it.Next() + _, _, valid = it.Current() + } + + return +} + +func (udc *UpsideDownCouch) Close() error { + return udc.store.Close() +} + +func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) { + // do analysis before acquiring write lock + analysisStart := time.Now() + numPlainTextBytes := doc.NumPlainTextBytes() + resultChan := make(chan *index.AnalysisResult) + aw := index.NewAnalysisWork(udc, doc, resultChan) + + // put the work on the queue + udc.analysisQueue.Queue(aw) + + // wait for the result + result := <-resultChan + close(resultChan) + atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) + + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + // open a reader for backindex lookup + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + return + } + + // first we lookup the backindex row for the doc id if it exists + // lookup the back index row + var backIndexRow *BackIndexRow + backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(doc.ID)) + if err != nil { + _ = kvreader.Close() + atomic.AddUint64(&udc.stats.errors, 1) + return + } + + err = kvreader.Close() + if err != nil { + return + } + + // start a writer for this update + indexStart := time.Now() + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := kvwriter.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + // prepare a list of rows + var addRowsAll [][]UpsideDownCouchRow + var updateRowsAll [][]UpsideDownCouchRow + var deleteRowsAll [][]UpsideDownCouchRow + + addRows, updateRows, deleteRows := udc.mergeOldAndNew(backIndexRow, result.Rows) + if len(addRows) > 0 { + addRowsAll = append(addRowsAll, addRows) + } + if len(updateRows) > 0 { + updateRowsAll = append(updateRowsAll, updateRows) + } + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + + err = udc.batchRows(kvwriter, addRowsAll, updateRowsAll, deleteRowsAll) + if err == nil && backIndexRow == nil { + udc.m.Lock() + udc.docCount++ + udc.m.Unlock() + } + atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) + if err == nil { + atomic.AddUint64(&udc.stats.updates, 1) + atomic.AddUint64(&udc.stats.numPlainTextBytesIndexed, numPlainTextBytes) + } else { + atomic.AddUint64(&udc.stats.errors, 1) + } + return +} + +func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows []index.IndexRow) (addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) { + addRows = make([]UpsideDownCouchRow, 0, len(rows)) + + if backIndexRow == nil { + addRows = addRows[0:len(rows)] + for i, row := range rows { + addRows[i] = row + } + return addRows, nil, nil + } + + updateRows = make([]UpsideDownCouchRow, 0, len(rows)) + deleteRows = make([]UpsideDownCouchRow, 0, len(rows)) + + var existingTermKeys map[string]struct{} + backIndexTermKeys := backIndexRow.AllTermKeys() + if len(backIndexTermKeys) > 0 { + existingTermKeys = make(map[string]struct{}, len(backIndexTermKeys)) + for _, key := range backIndexTermKeys { + existingTermKeys[string(key)] = struct{}{} + } + } + + var existingStoredKeys map[string]struct{} + backIndexStoredKeys := backIndexRow.AllStoredKeys() + if len(backIndexStoredKeys) > 0 { + existingStoredKeys = make(map[string]struct{}, len(backIndexStoredKeys)) + for _, key := range backIndexStoredKeys { + existingStoredKeys[string(key)] = struct{}{} + } + } + + keyBuf := GetRowBuffer() + for _, row := range rows { + switch row := row.(type) { + case *TermFrequencyRow: + if existingTermKeys != nil { + if row.KeySize() > len(keyBuf) { + keyBuf = make([]byte, row.KeySize()) + } + keySize, _ := row.KeyTo(keyBuf) + if _, ok := existingTermKeys[string(keyBuf[:keySize])]; ok { + updateRows = append(updateRows, row) + delete(existingTermKeys, string(keyBuf[:keySize])) + continue + } + } + addRows = append(addRows, row) + case *StoredRow: + if existingStoredKeys != nil { + if row.KeySize() > len(keyBuf) { + keyBuf = make([]byte, row.KeySize()) + } + keySize, _ := row.KeyTo(keyBuf) + if _, ok := existingStoredKeys[string(keyBuf[:keySize])]; ok { + updateRows = append(updateRows, row) + delete(existingStoredKeys, string(keyBuf[:keySize])) + continue + } + } + addRows = append(addRows, row) + default: + updateRows = append(updateRows, row) + } + } + PutRowBuffer(keyBuf) + + // any of the existing rows that weren't updated need to be deleted + for existingTermKey := range existingTermKeys { + termFreqRow, err := NewTermFrequencyRowK([]byte(existingTermKey)) + if err == nil { + deleteRows = append(deleteRows, termFreqRow) + } + } + + // any of the existing stored fields that weren't updated need to be deleted + for existingStoredKey := range existingStoredKeys { + storedRow, err := NewStoredRowK([]byte(existingStoredKey)) + if err == nil { + deleteRows = append(deleteRows, storedRow) + } + } + + return addRows, updateRows, deleteRows +} + +func (udc *UpsideDownCouch) storeField(docID []byte, field document.Field, fieldIndex uint16, rows []index.IndexRow, backIndexStoredEntries []*BackIndexStoreEntry) ([]index.IndexRow, []*BackIndexStoreEntry) { + fieldType := encodeFieldType(field) + storedRow := NewStoredRow(docID, fieldIndex, field.ArrayPositions(), fieldType, field.Value()) + + // record the back index entry + backIndexStoredEntry := BackIndexStoreEntry{Field: proto.Uint32(uint32(fieldIndex)), ArrayPositions: field.ArrayPositions()} + + return append(rows, storedRow), append(backIndexStoredEntries, &backIndexStoredEntry) +} + +func encodeFieldType(f document.Field) byte { + fieldType := byte('x') + switch f.(type) { + case *document.TextField: + fieldType = 't' + case *document.NumericField: + fieldType = 'n' + case *document.DateTimeField: + fieldType = 'd' + case *document.BooleanField: + fieldType = 'b' + case *document.GeoPointField: + fieldType = 'g' + case *document.CompositeField: + fieldType = 'c' + } + return fieldType +} + +func (udc *UpsideDownCouch) indexField(docID []byte, includeTermVectors bool, fieldIndex uint16, fieldLength int, tokenFreqs analysis.TokenFrequencies, rows []index.IndexRow, backIndexTermsEntries []*BackIndexTermsEntry) ([]index.IndexRow, []*BackIndexTermsEntry) { + fieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength))) + + termFreqRows := make([]TermFrequencyRow, len(tokenFreqs)) + termFreqRowsUsed := 0 + + terms := make([]string, 0, len(tokenFreqs)) + for k, tf := range tokenFreqs { + termFreqRow := &termFreqRows[termFreqRowsUsed] + termFreqRowsUsed++ + + InitTermFrequencyRow(termFreqRow, tf.Term, fieldIndex, docID, + uint64(frequencyFromTokenFreq(tf)), fieldNorm) + + if includeTermVectors { + termFreqRow.vectors, rows = udc.termVectorsFromTokenFreq(fieldIndex, tf, rows) + } + + // record the back index entry + terms = append(terms, k) + + rows = append(rows, termFreqRow) + } + backIndexTermsEntry := BackIndexTermsEntry{Field: proto.Uint32(uint32(fieldIndex)), Terms: terms} + backIndexTermsEntries = append(backIndexTermsEntries, &backIndexTermsEntry) + + return rows, backIndexTermsEntries +} + +func (udc *UpsideDownCouch) Delete(id string) (err error) { + indexStart := time.Now() + + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + // open a reader for backindex lookup + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + return + } + + // first we lookup the backindex row for the doc id if it exists + // lookup the back index row + var backIndexRow *BackIndexRow + backIndexRow, err = backIndexRowForDoc(kvreader, index.IndexInternalID(id)) + if err != nil { + _ = kvreader.Close() + atomic.AddUint64(&udc.stats.errors, 1) + return + } + + err = kvreader.Close() + if err != nil { + return + } + + if backIndexRow == nil { + atomic.AddUint64(&udc.stats.deletes, 1) + return + } + + // start a writer for this delete + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := kvwriter.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + var deleteRowsAll [][]UpsideDownCouchRow + + deleteRows := udc.deleteSingle(id, backIndexRow, nil) + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + + err = udc.batchRows(kvwriter, nil, nil, deleteRowsAll) + if err == nil { + udc.m.Lock() + udc.docCount-- + udc.m.Unlock() + } + atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) + if err == nil { + atomic.AddUint64(&udc.stats.deletes, 1) + } else { + atomic.AddUint64(&udc.stats.errors, 1) + } + return +} + +func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, deleteRows []UpsideDownCouchRow) []UpsideDownCouchRow { + idBytes := []byte(id) + + for _, backIndexEntry := range backIndexRow.termsEntries { + for i := range backIndexEntry.Terms { + tfr := NewTermFrequencyRow([]byte(backIndexEntry.Terms[i]), uint16(*backIndexEntry.Field), idBytes, 0, 0) + deleteRows = append(deleteRows, tfr) + } + } + for _, se := range backIndexRow.storedEntries { + sf := NewStoredRow(idBytes, uint16(*se.Field), se.ArrayPositions, 'x', nil) + deleteRows = append(deleteRows, sf) + } + + // also delete the back entry itself + deleteRows = append(deleteRows, backIndexRow) + return deleteRows +} + +func decodeFieldType(typ byte, name string, pos []uint64, value []byte) document.Field { + switch typ { + case 't': + return document.NewTextField(name, pos, value) + case 'n': + return document.NewNumericFieldFromBytes(name, pos, value) + case 'd': + return document.NewDateTimeFieldFromBytes(name, pos, value) + case 'b': + return document.NewBooleanFieldFromBytes(name, pos, value) + case 'g': + return document.NewGeoPointFieldFromBytes(name, pos, value) + } + return nil +} + +func frequencyFromTokenFreq(tf *analysis.TokenFreq) int { + return tf.Frequency() +} + +func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *analysis.TokenFreq, rows []index.IndexRow) ([]*TermVector, []index.IndexRow) { + a := make([]TermVector, len(tf.Locations)) + rv := make([]*TermVector, len(tf.Locations)) + + for i, l := range tf.Locations { + var newFieldRow *FieldRow + fieldIndex := field + if l.Field != "" { + // lookup correct field + fieldIndex, newFieldRow = udc.fieldIndexOrNewRow(l.Field) + if newFieldRow != nil { + rows = append(rows, newFieldRow) + } + } + a[i] = TermVector{ + field: fieldIndex, + arrayPositions: l.ArrayPositions, + pos: uint64(l.Position), + start: uint64(l.Start), + end: uint64(l.End), + } + rv[i] = &a[i] + } + + return rv, rows +} + +func (udc *UpsideDownCouch) termFieldVectorsFromTermVectors(in []*TermVector) []*index.TermFieldVector { + if len(in) <= 0 { + return nil + } + + a := make([]index.TermFieldVector, len(in)) + rv := make([]*index.TermFieldVector, len(in)) + + for i, tv := range in { + fieldName := udc.fieldCache.FieldIndexed(tv.field) + a[i] = index.TermFieldVector{ + Field: fieldName, + ArrayPositions: tv.arrayPositions, + Pos: tv.pos, + Start: tv.start, + End: tv.end, + } + rv[i] = &a[i] + } + return rv +} + +func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { + analysisStart := time.Now() + + resultChan := make(chan *index.AnalysisResult, len(batch.IndexOps)) + + var numUpdates uint64 + var numPlainTextBytes uint64 + for _, doc := range batch.IndexOps { + if doc != nil { + numUpdates++ + numPlainTextBytes += doc.NumPlainTextBytes() + } + } + + go func() { + for _, doc := range batch.IndexOps { + if doc != nil { + aw := index.NewAnalysisWork(udc, doc, resultChan) + // put the work on the queue + udc.analysisQueue.Queue(aw) + } + } + }() + + // retrieve back index rows concurrent with analysis + docBackIndexRowErr := error(nil) + docBackIndexRowCh := make(chan *docBackIndexRow, len(batch.IndexOps)) + + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + + go func() { + defer close(docBackIndexRowCh) + + // open a reader for backindex lookup + var kvreader store.KVReader + kvreader, err = udc.store.Reader() + if err != nil { + docBackIndexRowErr = err + return + } + + for docID, doc := range batch.IndexOps { + backIndexRow, err := backIndexRowForDoc(kvreader, index.IndexInternalID(docID)) + if err != nil { + docBackIndexRowErr = err + return + } + + docBackIndexRowCh <- &docBackIndexRow{docID, doc, backIndexRow} + } + + err = kvreader.Close() + if err != nil { + docBackIndexRowErr = err + return + } + }() + + // wait for analysis result + newRowsMap := make(map[string][]index.IndexRow) + var itemsDeQueued uint64 + for itemsDeQueued < numUpdates { + result := <-resultChan + newRowsMap[result.DocID] = result.Rows + itemsDeQueued++ + } + close(resultChan) + + atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart))) + + docsAdded := uint64(0) + docsDeleted := uint64(0) + + indexStart := time.Now() + + // prepare a list of rows + var addRowsAll [][]UpsideDownCouchRow + var updateRowsAll [][]UpsideDownCouchRow + var deleteRowsAll [][]UpsideDownCouchRow + + // add the internal ops + var updateRows []UpsideDownCouchRow + var deleteRows []UpsideDownCouchRow + + for internalKey, internalValue := range batch.InternalOps { + if internalValue == nil { + // delete + deleteInternalRow := NewInternalRow([]byte(internalKey), nil) + deleteRows = append(deleteRows, deleteInternalRow) + } else { + updateInternalRow := NewInternalRow([]byte(internalKey), internalValue) + updateRows = append(updateRows, updateInternalRow) + } + } + + if len(updateRows) > 0 { + updateRowsAll = append(updateRowsAll, updateRows) + } + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + + // process back index rows as they arrive + for dbir := range docBackIndexRowCh { + if dbir.doc == nil && dbir.backIndexRow != nil { + // delete + deleteRows := udc.deleteSingle(dbir.docID, dbir.backIndexRow, nil) + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + docsDeleted++ + } else if dbir.doc != nil { + addRows, updateRows, deleteRows := udc.mergeOldAndNew(dbir.backIndexRow, newRowsMap[dbir.docID]) + if len(addRows) > 0 { + addRowsAll = append(addRowsAll, addRows) + } + if len(updateRows) > 0 { + updateRowsAll = append(updateRowsAll, updateRows) + } + if len(deleteRows) > 0 { + deleteRowsAll = append(deleteRowsAll, deleteRows) + } + if dbir.backIndexRow == nil { + docsAdded++ + } + } + } + + if docBackIndexRowErr != nil { + return docBackIndexRowErr + } + + // start a writer for this batch + var kvwriter store.KVWriter + kvwriter, err = udc.store.Writer() + if err != nil { + return + } + + err = udc.batchRows(kvwriter, addRowsAll, updateRowsAll, deleteRowsAll) + if err != nil { + _ = kvwriter.Close() + atomic.AddUint64(&udc.stats.errors, 1) + return + } + + err = kvwriter.Close() + + atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart))) + + if err == nil { + udc.m.Lock() + udc.docCount += docsAdded + udc.docCount -= docsDeleted + udc.m.Unlock() + atomic.AddUint64(&udc.stats.updates, numUpdates) + atomic.AddUint64(&udc.stats.deletes, docsDeleted) + atomic.AddUint64(&udc.stats.batches, 1) + atomic.AddUint64(&udc.stats.numPlainTextBytesIndexed, numPlainTextBytes) + } else { + atomic.AddUint64(&udc.stats.errors, 1) + } + return +} + +func (udc *UpsideDownCouch) SetInternal(key, val []byte) (err error) { + internalRow := NewInternalRow(key, val) + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + var writer store.KVWriter + writer, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := writer.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + batch := writer.NewBatch() + batch.Set(internalRow.Key(), internalRow.Value()) + + return writer.ExecuteBatch(batch) +} + +func (udc *UpsideDownCouch) DeleteInternal(key []byte) (err error) { + internalRow := NewInternalRow(key, nil) + udc.writeMutex.Lock() + defer udc.writeMutex.Unlock() + var writer store.KVWriter + writer, err = udc.store.Writer() + if err != nil { + return + } + defer func() { + if cerr := writer.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + batch := writer.NewBatch() + batch.Delete(internalRow.Key()) + return writer.ExecuteBatch(batch) +} + +func (udc *UpsideDownCouch) Reader() (index.IndexReader, error) { + kvr, err := udc.store.Reader() + if err != nil { + return nil, fmt.Errorf("error opening store reader: %v", err) + } + udc.m.RLock() + defer udc.m.RUnlock() + return &IndexReader{ + index: udc, + kvreader: kvr, + docCount: udc.docCount, + }, nil +} + +func (udc *UpsideDownCouch) Stats() json.Marshaler { + return udc.stats +} + +func (udc *UpsideDownCouch) StatsMap() map[string]interface{} { + return udc.stats.statsMap() +} + +func (udc *UpsideDownCouch) Advanced() (store.KVStore, error) { + return udc.store, nil +} + +func (udc *UpsideDownCouch) fieldIndexOrNewRow(name string) (uint16, *FieldRow) { + index, existed := udc.fieldCache.FieldNamed(name, true) + if !existed { + return index, NewFieldRow(index, name) + } + return index, nil +} + +func init() { + registry.RegisterIndexType(Name, NewUpsideDownCouch) +} + +func backIndexRowForDoc(kvreader store.KVReader, docID index.IndexInternalID) (*BackIndexRow, error) { + // use a temporary row structure to build key + tempRow := BackIndexRow{ + doc: docID, + } + + keyBuf := GetRowBuffer() + if tempRow.KeySize() > len(keyBuf) { + keyBuf = make([]byte, 2*tempRow.KeySize()) + } + defer PutRowBuffer(keyBuf) + keySize, err := tempRow.KeyTo(keyBuf) + if err != nil { + return nil, err + } + + value, err := kvreader.Get(keyBuf[:keySize]) + if err != nil { + return nil, err + } + if value == nil { + return nil, nil + } + backIndexRow, err := NewBackIndexRowKV(keyBuf[:keySize], value) + if err != nil { + return nil, err + } + return backIndexRow, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go new file mode 100644 index 00000000..c161e1cc --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.pb.go @@ -0,0 +1,688 @@ +// Code generated by protoc-gen-gogo. +// source: upsidedown.proto +// DO NOT EDIT! + +/* + Package upsidedown is a generated protocol buffer package. + + It is generated from these files: + upsidedown.proto + + It has these top-level messages: + BackIndexTermsEntry + BackIndexStoreEntry + BackIndexRowValue +*/ +package upsidedown + +import proto "github.com/golang/protobuf/proto" +import math "math" + +import io "io" +import fmt "fmt" +import github_com_golang_protobuf_proto "github.com/golang/protobuf/proto" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = math.Inf + +type BackIndexTermsEntry struct { + Field *uint32 `protobuf:"varint,1,req,name=field" json:"field,omitempty"` + Terms []string `protobuf:"bytes,2,rep,name=terms" json:"terms,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *BackIndexTermsEntry) Reset() { *m = BackIndexTermsEntry{} } +func (m *BackIndexTermsEntry) String() string { return proto.CompactTextString(m) } +func (*BackIndexTermsEntry) ProtoMessage() {} + +func (m *BackIndexTermsEntry) GetField() uint32 { + if m != nil && m.Field != nil { + return *m.Field + } + return 0 +} + +func (m *BackIndexTermsEntry) GetTerms() []string { + if m != nil { + return m.Terms + } + return nil +} + +type BackIndexStoreEntry struct { + Field *uint32 `protobuf:"varint,1,req,name=field" json:"field,omitempty"` + ArrayPositions []uint64 `protobuf:"varint,2,rep,name=arrayPositions" json:"arrayPositions,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *BackIndexStoreEntry) Reset() { *m = BackIndexStoreEntry{} } +func (m *BackIndexStoreEntry) String() string { return proto.CompactTextString(m) } +func (*BackIndexStoreEntry) ProtoMessage() {} + +func (m *BackIndexStoreEntry) GetField() uint32 { + if m != nil && m.Field != nil { + return *m.Field + } + return 0 +} + +func (m *BackIndexStoreEntry) GetArrayPositions() []uint64 { + if m != nil { + return m.ArrayPositions + } + return nil +} + +type BackIndexRowValue struct { + TermsEntries []*BackIndexTermsEntry `protobuf:"bytes,1,rep,name=termsEntries" json:"termsEntries,omitempty"` + StoredEntries []*BackIndexStoreEntry `protobuf:"bytes,2,rep,name=storedEntries" json:"storedEntries,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *BackIndexRowValue) Reset() { *m = BackIndexRowValue{} } +func (m *BackIndexRowValue) String() string { return proto.CompactTextString(m) } +func (*BackIndexRowValue) ProtoMessage() {} + +func (m *BackIndexRowValue) GetTermsEntries() []*BackIndexTermsEntry { + if m != nil { + return m.TermsEntries + } + return nil +} + +func (m *BackIndexRowValue) GetStoredEntries() []*BackIndexStoreEntry { + if m != nil { + return m.StoredEntries + } + return nil +} + +func (m *BackIndexTermsEntry) Unmarshal(data []byte) error { + var hasFields [1]uint64 + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Field = &v + hasFields[0] |= uint64(0x00000001) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Terms", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + int(stringLen) + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Terms = append(m.Terms, string(data[iNdEx:postIndex])) + iNdEx = postIndex + default: + var sizeOfWire int + for { + sizeOfWire++ + wire >>= 7 + if wire == 0 { + break + } + } + iNdEx -= sizeOfWire + skippy, err := skipUpsidedown(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthUpsidedown + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + if hasFields[0]&uint64(0x00000001) == 0 { + return new(github_com_golang_protobuf_proto.RequiredNotSetError) + } + + return nil +} +func (m *BackIndexStoreEntry) Unmarshal(data []byte) error { + var hasFields [1]uint64 + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Field", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Field = &v + hasFields[0] |= uint64(0x00000001) + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ArrayPositions", wireType) + } + var v uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ArrayPositions = append(m.ArrayPositions, v) + default: + var sizeOfWire int + for { + sizeOfWire++ + wire >>= 7 + if wire == 0 { + break + } + } + iNdEx -= sizeOfWire + skippy, err := skipUpsidedown(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthUpsidedown + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + if hasFields[0]&uint64(0x00000001) == 0 { + return new(github_com_golang_protobuf_proto.RequiredNotSetError) + } + + return nil +} +func (m *BackIndexRowValue) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TermsEntries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + msglen + if msglen < 0 { + return ErrInvalidLengthUpsidedown + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TermsEntries = append(m.TermsEntries, &BackIndexTermsEntry{}) + if err := m.TermsEntries[len(m.TermsEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StoredEntries", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + postIndex := iNdEx + msglen + if msglen < 0 { + return ErrInvalidLengthUpsidedown + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StoredEntries = append(m.StoredEntries, &BackIndexStoreEntry{}) + if err := m.StoredEntries[len(m.StoredEntries)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + var sizeOfWire int + for { + sizeOfWire++ + wire >>= 7 + if wire == 0 { + break + } + } + iNdEx -= sizeOfWire + skippy, err := skipUpsidedown(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthUpsidedown + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + return nil +} +func skipUpsidedown(data []byte) (n int, err error) { + l := len(data) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if data[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthUpsidedown + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipUpsidedown(data[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthUpsidedown = fmt.Errorf("proto: negative length found during unmarshaling") +) + +func (m *BackIndexTermsEntry) Size() (n int) { + var l int + _ = l + if m.Field != nil { + n += 1 + sovUpsidedown(uint64(*m.Field)) + } + if len(m.Terms) > 0 { + for _, s := range m.Terms { + l = len(s) + n += 1 + l + sovUpsidedown(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *BackIndexStoreEntry) Size() (n int) { + var l int + _ = l + if m.Field != nil { + n += 1 + sovUpsidedown(uint64(*m.Field)) + } + if len(m.ArrayPositions) > 0 { + for _, e := range m.ArrayPositions { + n += 1 + sovUpsidedown(uint64(e)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *BackIndexRowValue) Size() (n int) { + var l int + _ = l + if len(m.TermsEntries) > 0 { + for _, e := range m.TermsEntries { + l = e.Size() + n += 1 + l + sovUpsidedown(uint64(l)) + } + } + if len(m.StoredEntries) > 0 { + for _, e := range m.StoredEntries { + l = e.Size() + n += 1 + l + sovUpsidedown(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovUpsidedown(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozUpsidedown(x uint64) (n int) { + return sovUpsidedown(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *BackIndexTermsEntry) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *BackIndexTermsEntry) MarshalTo(data []byte) (n int, err error) { + var i int + _ = i + var l int + _ = l + if m.Field == nil { + return 0, new(github_com_golang_protobuf_proto.RequiredNotSetError) + } else { + data[i] = 0x8 + i++ + i = encodeVarintUpsidedown(data, i, uint64(*m.Field)) + } + if len(m.Terms) > 0 { + for _, s := range m.Terms { + data[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + data[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + data[i] = uint8(l) + i++ + i += copy(data[i:], s) + } + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *BackIndexStoreEntry) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *BackIndexStoreEntry) MarshalTo(data []byte) (n int, err error) { + var i int + _ = i + var l int + _ = l + if m.Field == nil { + return 0, new(github_com_golang_protobuf_proto.RequiredNotSetError) + } else { + data[i] = 0x8 + i++ + i = encodeVarintUpsidedown(data, i, uint64(*m.Field)) + } + if len(m.ArrayPositions) > 0 { + for _, num := range m.ArrayPositions { + data[i] = 0x10 + i++ + i = encodeVarintUpsidedown(data, i, uint64(num)) + } + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *BackIndexRowValue) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *BackIndexRowValue) MarshalTo(data []byte) (n int, err error) { + var i int + _ = i + var l int + _ = l + if len(m.TermsEntries) > 0 { + for _, msg := range m.TermsEntries { + data[i] = 0xa + i++ + i = encodeVarintUpsidedown(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.StoredEntries) > 0 { + for _, msg := range m.StoredEntries { + data[i] = 0x12 + i++ + i = encodeVarintUpsidedown(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.XXX_unrecognized != nil { + i += copy(data[i:], m.XXX_unrecognized) + } + return i, nil +} + +func encodeFixed64Upsidedown(data []byte, offset int, v uint64) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + data[offset+4] = uint8(v >> 32) + data[offset+5] = uint8(v >> 40) + data[offset+6] = uint8(v >> 48) + data[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Upsidedown(data []byte, offset int, v uint32) int { + data[offset] = uint8(v) + data[offset+1] = uint8(v >> 8) + data[offset+2] = uint8(v >> 16) + data[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintUpsidedown(data []byte, offset int, v uint64) int { + for v >= 1<<7 { + data[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + data[offset] = uint8(v) + return offset + 1 +} diff --git a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.proto b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto similarity index 56% rename from vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.proto rename to vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto index 27f24ccb..cf0492a2 100644 --- a/vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.proto +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.proto @@ -1,6 +1,6 @@ -message BackIndexTermEntry { - required string term = 1; - required uint32 field = 2; +message BackIndexTermsEntry { + required uint32 field = 1; + repeated string terms = 2; } message BackIndexStoreEntry { @@ -9,6 +9,6 @@ message BackIndexStoreEntry { } message BackIndexRowValue { - repeated BackIndexTermEntry termEntries = 1; + repeated BackIndexTermsEntry termsEntries = 1; repeated BackIndexStoreEntry storedEntries = 2; -} \ No newline at end of file +} diff --git a/vendor/github.com/blevesearch/bleve/index_alias.go b/vendor/github.com/blevesearch/bleve/index_alias.go index c46be279..7a85d721 100644 --- a/vendor/github.com/blevesearch/bleve/index_alias.go +++ b/vendor/github.com/blevesearch/bleve/index_alias.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve diff --git a/vendor/github.com/blevesearch/bleve/index_alias_impl.go b/vendor/github.com/blevesearch/bleve/index_alias_impl.go index a11f330b..9e9a3594 100644 --- a/vendor/github.com/blevesearch/bleve/index_alias_impl.go +++ b/vendor/github.com/blevesearch/bleve/index_alias_impl.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve @@ -14,13 +19,17 @@ import ( "sync" "time" + "golang.org/x/net/context" + "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/mapping" "github.com/blevesearch/bleve/search" ) type indexAliasImpl struct { + name string indexes []Index mutex sync.RWMutex open bool @@ -30,6 +39,7 @@ type indexAliasImpl struct { // Index objects. func NewIndexAlias(indexes ...Index) *indexAliasImpl { return &indexAliasImpl{ + name: "alias", indexes: indexes, open: true, } @@ -120,16 +130,20 @@ func (i *indexAliasImpl) DocCount() (uint64, error) { for _, index := range i.indexes { otherCount, err := index.DocCount() - if err != nil { - return 0, err + if err == nil { + rv += otherCount } - rv += otherCount + // tolerate errors to produce partial counts } return rv, nil } func (i *indexAliasImpl) Search(req *SearchRequest) (*SearchResult, error) { + return i.SearchInContext(context.Background(), req) +} + +func (i *indexAliasImpl) SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -143,10 +157,10 @@ func (i *indexAliasImpl) Search(req *SearchRequest) (*SearchResult, error) { // short circuit the simple case if len(i.indexes) == 1 { - return i.indexes[0].Search(req) + return i.indexes[0].SearchInContext(ctx, req) } - return MultiSearch(req, i.indexes...) + return MultiSearch(ctx, req, i.indexes...) } func (i *indexAliasImpl) Fields() ([]string, error) { @@ -243,54 +257,6 @@ func (i *indexAliasImpl) FieldDictPrefix(field string, termPrefix []byte) (index }, nil } -func (i *indexAliasImpl) DumpAll() chan interface{} { - i.mutex.RLock() - defer i.mutex.RUnlock() - - if !i.open { - return nil - } - - err := i.isAliasToSingleIndex() - if err != nil { - return nil - } - - return i.indexes[0].DumpAll() -} - -func (i *indexAliasImpl) DumpDoc(id string) chan interface{} { - i.mutex.RLock() - defer i.mutex.RUnlock() - - if !i.open { - return nil - } - - err := i.isAliasToSingleIndex() - if err != nil { - return nil - } - - return i.indexes[0].DumpDoc(id) -} - -func (i *indexAliasImpl) DumpFields() chan interface{} { - i.mutex.RLock() - defer i.mutex.RUnlock() - - if !i.open { - return nil - } - - err := i.isAliasToSingleIndex() - if err != nil { - return nil - } - - return i.indexes[0].DumpFields() -} - func (i *indexAliasImpl) Close() error { i.mutex.Lock() defer i.mutex.Unlock() @@ -299,7 +265,7 @@ func (i *indexAliasImpl) Close() error { return nil } -func (i *indexAliasImpl) Mapping() *IndexMapping { +func (i *indexAliasImpl) Mapping() mapping.IndexMapping { i.mutex.RLock() defer i.mutex.RUnlock() @@ -331,6 +297,22 @@ func (i *indexAliasImpl) Stats() *IndexStat { return i.indexes[0].Stats() } +func (i *indexAliasImpl) StatsMap() map[string]interface{} { + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return nil + } + + err := i.isAliasToSingleIndex() + if err != nil { + return nil + } + + return i.indexes[0].StatsMap() +} + func (i *indexAliasImpl) GetInternal(key []byte) ([]byte, error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -436,89 +418,94 @@ func (i *indexAliasImpl) Swap(in, out []Index) { // createChildSearchRequest creates a separate // request from the original // For now, avoid data race on req structure. -// TODO disable highligh/field load on child +// TODO disable highlight/field load on child // requests, and add code to do this only on // the actual final results. // Perhaps that part needs to be optional, // could be slower in remote usages. func createChildSearchRequest(req *SearchRequest) *SearchRequest { rv := SearchRequest{ - Query: req.Query, - Size: req.Size + req.From, - From: 0, - Highlight: req.Highlight, - Fields: req.Fields, - Facets: req.Facets, - Explain: req.Explain, + Query: req.Query, + Size: req.Size + req.From, + From: 0, + Highlight: req.Highlight, + Fields: req.Fields, + Facets: req.Facets, + Explain: req.Explain, + Sort: req.Sort.Copy(), + IncludeLocations: req.IncludeLocations, } return &rv } -// MultiSearch executes a SearchRequest across multiple -// Index objects, then merges the results. -func MultiSearch(req *SearchRequest, indexes ...Index) (*SearchResult, error) { +type asyncSearchResult struct { + Name string + Result *SearchResult + Err error +} + +// MultiSearch executes a SearchRequest across multiple Index objects, +// then merges the results. The indexes must honor any ctx deadline. +func MultiSearch(ctx context.Context, req *SearchRequest, indexes ...Index) (*SearchResult, error) { + searchStart := time.Now() - results := make(chan *SearchResult) - errs := make(chan error) + asyncResults := make(chan *asyncSearchResult, len(indexes)) // run search on each index in separate go routine var waitGroup sync.WaitGroup - var searchChildIndex = func(waitGroup *sync.WaitGroup, in Index, results chan *SearchResult, errs chan error) { - go func() { - defer waitGroup.Done() - childReq := createChildSearchRequest(req) - searchResult, err := in.Search(childReq) - if err != nil { - errs <- err - } else { - results <- searchResult - } - }() + var searchChildIndex = func(in Index, childReq *SearchRequest) { + rv := asyncSearchResult{Name: in.Name()} + rv.Result, rv.Err = in.SearchInContext(ctx, childReq) + asyncResults <- &rv + waitGroup.Done() } + waitGroup.Add(len(indexes)) for _, in := range indexes { - waitGroup.Add(1) - searchChildIndex(&waitGroup, in, results, errs) + go searchChildIndex(in, createChildSearchRequest(req)) } // on another go routine, close after finished go func() { waitGroup.Wait() - close(results) - close(errs) + close(asyncResults) }() var sr *SearchResult - var err error - var result *SearchResult - ok := true - for ok { - select { - case result, ok = <-results: - if ok { - if sr == nil { - // first result - sr = result - } else { - // merge with previous - sr.Merge(result) - } - } - case err, ok = <-errs: - // for now stop on any error - // FIXME offer other behaviors - if err != nil { - return nil, err + indexErrors := make(map[string]error) + + for asr := range asyncResults { + if asr.Err == nil { + if sr == nil { + // first result + sr = asr.Result + } else { + // merge with previous + sr.Merge(asr.Result) } + } else { + indexErrors[asr.Name] = asr.Err } } // merge just concatenated all the hits // now lets clean it up - // first sort it by score - sort.Sort(sr.Hits) + // handle case where no results were successful + if sr == nil { + sr = &SearchResult{ + Status: &SearchStatus{ + Errors: make(map[string]error), + }, + } + } + + // sort all hits with the requested order + if len(req.Sort) > 0 { + sorter := newMultiSearchHitSorter(req.Sort, sr.Hits) + sort.Sort(sorter) + } // now skip over the correct From if req.From > 0 && len(sr.Hits) > req.From { @@ -542,6 +529,18 @@ func MultiSearch(req *SearchRequest, indexes ...Index) (*SearchResult, error) { searchDuration := time.Since(searchStart) sr.Took = searchDuration + // fix up errors + if len(indexErrors) > 0 { + if sr.Status.Errors == nil { + sr.Status.Errors = make(map[string]error) + } + for indexName, indexErr := range indexErrors { + sr.Status.Errors[indexName] = indexErr + sr.Status.Total++ + sr.Status.Failed++ + } + } + return sr, nil } @@ -561,6 +560,14 @@ func (i *indexAliasImpl) NewBatch() *Batch { return i.indexes[0].NewBatch() } +func (i *indexAliasImpl) Name() string { + return i.name +} + +func (i *indexAliasImpl) SetName(name string) { + i.name = name +} + type indexAliasImplFieldDict struct { index *indexAliasImpl fieldDict index.FieldDict @@ -574,3 +581,26 @@ func (f *indexAliasImplFieldDict) Close() error { defer f.index.mutex.RUnlock() return f.fieldDict.Close() } + +type multiSearchHitSorter struct { + hits search.DocumentMatchCollection + sort search.SortOrder + cachedScoring []bool + cachedDesc []bool +} + +func newMultiSearchHitSorter(sort search.SortOrder, hits search.DocumentMatchCollection) *multiSearchHitSorter { + return &multiSearchHitSorter{ + sort: sort, + hits: hits, + cachedScoring: sort.CacheIsScore(), + cachedDesc: sort.CacheDescending(), + } +} + +func (m *multiSearchHitSorter) Len() int { return len(m.hits) } +func (m *multiSearchHitSorter) Swap(i, j int) { m.hits[i], m.hits[j] = m.hits[j], m.hits[i] } +func (m *multiSearchHitSorter) Less(i, j int) bool { + c := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j]) + return c < 0 +} diff --git a/vendor/github.com/blevesearch/bleve/index_alias_impl_test.go b/vendor/github.com/blevesearch/bleve/index_alias_impl_test.go deleted file mode 100644 index 862fb2d9..00000000 --- a/vendor/github.com/blevesearch/bleve/index_alias_impl_test.go +++ /dev/null @@ -1,790 +0,0 @@ -package bleve - -import ( - "fmt" - "reflect" - "testing" - "time" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/search" -) - -func TestIndexAliasSingle(t *testing.T) { - expectedError := fmt.Errorf("expected") - ei1 := &stubIndex{ - err: expectedError, - } - - alias := NewIndexAlias(ei1) - - err := alias.Index("a", "a") - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - err = alias.Delete("a") - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - batch := alias.NewBatch() - err = alias.Batch(batch) - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - _, err = alias.Document("a") - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - _, err = alias.Fields() - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - _, err = alias.GetInternal([]byte("a")) - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - err = alias.SetInternal([]byte("a"), []byte("a")) - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - err = alias.DeleteInternal([]byte("a")) - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - res := alias.DumpAll() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpDoc("a") - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpFields() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - mapping := alias.Mapping() - if mapping != nil { - t.Errorf("expected nil, got %v", res) - } - - indexStat := alias.Stats() - if indexStat != nil { - t.Errorf("expected nil, got %v", res) - } - - // now a few things that should work - sr := NewSearchRequest(NewTermQuery("test")) - _, err = alias.Search(sr) - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - _, err = alias.DocCount() - if err != expectedError { - t.Errorf("expected %v, got %v", expectedError, err) - } - - // now change the def using add/remove - expectedError2 := fmt.Errorf("expected2") - ei2 := &stubIndex{ - err: expectedError2, - } - - alias.Add(ei2) - alias.Remove(ei1) - - err = alias.Index("a", "a") - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - err = alias.Delete("a") - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - err = alias.Batch(batch) - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - _, err = alias.Document("a") - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - _, err = alias.Fields() - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - _, err = alias.GetInternal([]byte("a")) - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - err = alias.SetInternal([]byte("a"), []byte("a")) - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - err = alias.DeleteInternal([]byte("a")) - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - res = alias.DumpAll() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpDoc("a") - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpFields() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - mapping = alias.Mapping() - if mapping != nil { - t.Errorf("expected nil, got %v", res) - } - - indexStat = alias.Stats() - if indexStat != nil { - t.Errorf("expected nil, got %v", res) - } - - // now a few things that should work - _, err = alias.Search(sr) - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - _, err = alias.DocCount() - if err != expectedError2 { - t.Errorf("expected %v, got %v", expectedError2, err) - } - - // now change the def using swap - expectedError3 := fmt.Errorf("expected3") - ei3 := &stubIndex{ - err: expectedError3, - } - - alias.Swap([]Index{ei3}, []Index{ei2}) - - err = alias.Index("a", "a") - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - err = alias.Delete("a") - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - err = alias.Batch(batch) - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - _, err = alias.Document("a") - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - _, err = alias.Fields() - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - _, err = alias.GetInternal([]byte("a")) - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - err = alias.SetInternal([]byte("a"), []byte("a")) - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - err = alias.DeleteInternal([]byte("a")) - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - res = alias.DumpAll() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpDoc("a") - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpFields() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - mapping = alias.Mapping() - if mapping != nil { - t.Errorf("expected nil, got %v", res) - } - - indexStat = alias.Stats() - if indexStat != nil { - t.Errorf("expected nil, got %v", res) - } - - // now a few things that should work - _, err = alias.Search(sr) - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } - - _, err = alias.DocCount() - if err != expectedError3 { - t.Errorf("expected %v, got %v", expectedError3, err) - } -} - -func TestIndexAliasClosed(t *testing.T) { - alias := NewIndexAlias() - err := alias.Close() - if err != nil { - t.Fatal(err) - } - - err = alias.Index("a", "a") - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - err = alias.Delete("a") - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - batch := alias.NewBatch() - err = alias.Batch(batch) - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - _, err = alias.Document("a") - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - _, err = alias.Fields() - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - _, err = alias.GetInternal([]byte("a")) - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - err = alias.SetInternal([]byte("a"), []byte("a")) - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - err = alias.DeleteInternal([]byte("a")) - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - res := alias.DumpAll() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpDoc("a") - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpFields() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - mapping := alias.Mapping() - if mapping != nil { - t.Errorf("expected nil, got %v", res) - } - - indexStat := alias.Stats() - if indexStat != nil { - t.Errorf("expected nil, got %v", res) - } - - // now a few things that should work - sr := NewSearchRequest(NewTermQuery("test")) - _, err = alias.Search(sr) - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } - - _, err = alias.DocCount() - if err != ErrorIndexClosed { - t.Errorf("expected %v, got %v", ErrorIndexClosed, err) - } -} - -func TestIndexAliasEmpty(t *testing.T) { - alias := NewIndexAlias() - - err := alias.Index("a", "a") - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - err = alias.Delete("a") - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - batch := alias.NewBatch() - err = alias.Batch(batch) - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - _, err = alias.Document("a") - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - _, err = alias.Fields() - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - _, err = alias.GetInternal([]byte("a")) - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - err = alias.SetInternal([]byte("a"), []byte("a")) - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - err = alias.DeleteInternal([]byte("a")) - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - res := alias.DumpAll() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpDoc("a") - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpFields() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - mapping := alias.Mapping() - if mapping != nil { - t.Errorf("expected nil, got %v", res) - } - - indexStat := alias.Stats() - if indexStat != nil { - t.Errorf("expected nil, got %v", res) - } - - // now a few things that should work - sr := NewSearchRequest(NewTermQuery("test")) - _, err = alias.Search(sr) - if err != ErrorAliasEmpty { - t.Errorf("expected %v, got %v", ErrorAliasEmpty, err) - } - - count, err := alias.DocCount() - if count != 0 { - t.Errorf("expected %d, got %d", 0, count) - } -} - -func TestIndexAliasMulti(t *testing.T) { - ei1Count := uint64(7) - ei1 := &stubIndex{ - err: nil, - docCountResult: &ei1Count, - searchResult: &SearchResult{ - Total: 1, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", - Score: 1.0, - }, - }, - MaxScore: 1.0, - }} - ei2Count := uint64(8) - ei2 := &stubIndex{ - err: nil, - docCountResult: &ei2Count, - searchResult: &SearchResult{ - Total: 1, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "b", - Score: 2.0, - }, - }, - MaxScore: 2.0, - }} - - alias := NewIndexAlias(ei1, ei2) - - err := alias.Index("a", "a") - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - err = alias.Delete("a") - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - batch := alias.NewBatch() - err = alias.Batch(batch) - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - _, err = alias.Document("a") - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - _, err = alias.Fields() - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - _, err = alias.GetInternal([]byte("a")) - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - err = alias.SetInternal([]byte("a"), []byte("a")) - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - err = alias.DeleteInternal([]byte("a")) - if err != ErrorAliasMulti { - t.Errorf("expected %v, got %v", ErrorAliasMulti, err) - } - - res := alias.DumpAll() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpDoc("a") - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - res = alias.DumpFields() - if res != nil { - t.Errorf("expected nil, got %v", res) - } - - mapping := alias.Mapping() - if mapping != nil { - t.Errorf("expected nil, got %v", res) - } - - indexStat := alias.Stats() - if indexStat != nil { - t.Errorf("expected nil, got %v", res) - } - - // now a few things that should work - sr := NewSearchRequest(NewTermQuery("test")) - expected := &SearchResult{ - Request: sr, - Total: 2, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "b", - Score: 2.0, - }, - &search.DocumentMatch{ - ID: "a", - Score: 1.0, - }, - }, - MaxScore: 2.0, - } - results, err := alias.Search(sr) - if err != nil { - t.Error(err) - } - // cheat and ensure that Took field matches since it invovles time - expected.Took = results.Took - if !reflect.DeepEqual(results, expected) { - t.Errorf("expected %#v, got %#v", expected, results) - } - - count, err := alias.DocCount() - if count != (*ei1.docCountResult + *ei2.docCountResult) { - t.Errorf("expected %d, got %d", (*ei1.docCountResult + *ei2.docCountResult), count) - } -} - -// TestMultiSearchNoError -func TestMultiSearchNoError(t *testing.T) { - ei1 := &stubIndex{err: nil, searchResult: &SearchResult{ - Total: 1, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", - Score: 1.0, - }, - }, - MaxScore: 1.0, - }} - ei2 := &stubIndex{err: nil, searchResult: &SearchResult{ - Total: 1, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "b", - Score: 2.0, - }, - }, - MaxScore: 2.0, - }} - - sr := NewSearchRequest(NewTermQuery("test")) - expected := &SearchResult{ - Request: sr, - Total: 2, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "b", - Score: 2.0, - }, - &search.DocumentMatch{ - ID: "a", - Score: 1.0, - }, - }, - MaxScore: 2.0, - } - - results, err := MultiSearch(sr, ei1, ei2) - if err != nil { - t.Error(err) - } - // cheat and ensure that Took field matches since it invovles time - expected.Took = results.Took - if !reflect.DeepEqual(results, expected) { - t.Errorf("expected %#v, got %#v", expected, results) - } -} - -// TestMultiSearchSomeError -func TestMultiSearchSomeError(t *testing.T) { - ei1 := &stubIndex{err: nil, searchResult: &SearchResult{ - Total: 1, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", - Score: 1.0, - }, - }, - Took: 1 * time.Second, - MaxScore: 1.0, - }} - ei2 := &stubIndex{err: fmt.Errorf("deliberate error")} - sr := NewSearchRequest(NewTermQuery("test")) - _, err := MultiSearch(sr, ei1, ei2) - if err == nil { - t.Errorf("expected error, got %v", err) - } -} - -// TestMultiSearchAllError -// reproduces https://github.com/blevesearch/bleve/issues/126 -func TestMultiSearchAllError(t *testing.T) { - ei1 := &stubIndex{err: fmt.Errorf("deliberate error")} - ei2 := &stubIndex{err: fmt.Errorf("deliberate error")} - sr := NewSearchRequest(NewTermQuery("test")) - _, err := MultiSearch(sr, ei1, ei2) - if err == nil { - t.Errorf("expected error, got %v", err) - } -} - -func TestMultiSearchSecondPage(t *testing.T) { - checkRequest := func(sr *SearchRequest) error { - if sr.From != 0 { - return fmt.Errorf("child request from should be 0") - } - if sr.Size != 20 { - return fmt.Errorf("child request size should be 20") - } - return nil - } - - ei1 := &stubIndex{ - searchResult: &SearchResult{}, - checkRequest: checkRequest, - } - ei2 := &stubIndex{ - searchResult: &SearchResult{}, - checkRequest: checkRequest, - } - sr := NewSearchRequestOptions(NewTermQuery("test"), 10, 10, false) - _, err := MultiSearch(sr, ei1, ei2) - if err != nil { - t.Errorf("unexpected error %v", err) - } - -} - -// stubIndex is an Index impl for which all operations -// return the configured error value, unless the -// corresponding operation result value has been -// set, in which case that is returned instead -type stubIndex struct { - err error - searchResult *SearchResult - documentResult *document.Document - docCountResult *uint64 - checkRequest func(*SearchRequest) error -} - -func (i *stubIndex) Index(id string, data interface{}) error { - return i.err -} - -func (i *stubIndex) Delete(id string) error { - return i.err -} - -func (i *stubIndex) Batch(b *Batch) error { - return i.err -} - -func (i *stubIndex) Document(id string) (*document.Document, error) { - if i.documentResult != nil { - return i.documentResult, nil - } - return nil, i.err -} - -func (i *stubIndex) DocCount() (uint64, error) { - if i.docCountResult != nil { - return *i.docCountResult, nil - } - return 0, i.err -} - -func (i *stubIndex) Search(req *SearchRequest) (*SearchResult, error) { - if i.checkRequest != nil { - err := i.checkRequest(req) - if err != nil { - return nil, err - } - } - if i.searchResult != nil { - return i.searchResult, nil - } - return nil, i.err -} - -func (i *stubIndex) Fields() ([]string, error) { - return nil, i.err -} - -func (i *stubIndex) FieldDict(field string) (index.FieldDict, error) { - return nil, i.err -} - -func (i *stubIndex) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) { - return nil, i.err -} - -func (i *stubIndex) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) { - return nil, i.err -} - -func (i *stubIndex) DumpAll() chan interface{} { - return nil -} - -func (i *stubIndex) DumpDoc(id string) chan interface{} { - return nil -} - -func (i *stubIndex) DumpFields() chan interface{} { - return nil -} - -func (i *stubIndex) Close() error { - return i.err -} - -func (i *stubIndex) Mapping() *IndexMapping { - return nil -} - -func (i *stubIndex) Stats() *IndexStat { - return nil -} - -func (i *stubIndex) GetInternal(key []byte) ([]byte, error) { - return nil, i.err -} - -func (i *stubIndex) SetInternal(key, val []byte) error { - return i.err -} - -func (i *stubIndex) DeleteInternal(key []byte) error { - return i.err -} - -func (i *stubIndex) Advanced() (index.Index, store.KVStore, error) { - return nil, nil, nil -} - -func (i *stubIndex) NewBatch() *Batch { - return &Batch{} -} diff --git a/vendor/github.com/blevesearch/bleve/index_impl.go b/vendor/github.com/blevesearch/bleve/index_impl.go index c30563e0..c1c4608c 100644 --- a/vendor/github.com/blevesearch/bleve/index_impl.go +++ b/vendor/github.com/blevesearch/bleve/index_impl.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve @@ -17,22 +22,26 @@ import ( "sync/atomic" "time" + "golang.org/x/net/context" + "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/upside_down" + "github.com/blevesearch/bleve/index/upsidedown" + "github.com/blevesearch/bleve/mapping" "github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/collectors" - "github.com/blevesearch/bleve/search/facets" + "github.com/blevesearch/bleve/search/collector" + "github.com/blevesearch/bleve/search/facet" + "github.com/blevesearch/bleve/search/highlight" ) type indexImpl struct { path string + name string meta *indexMeta - s store.KVStore i index.Index - m *IndexMapping + m mapping.IndexMapping mutex sync.RWMutex open bool stats *IndexStat @@ -46,97 +55,58 @@ func indexStorePath(path string) string { return path + string(os.PathSeparator) + storePath } -func newMemIndex(mapping *IndexMapping) (*indexImpl, error) { - rv := indexImpl{ - path: "", - m: mapping, - meta: newIndexMeta("mem", nil), - stats: &IndexStat{}, - } - - storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) - if storeConstructor == nil { - return nil, ErrorUnknownStorageType - } - // now open the store - var err error - rv.s, err = storeConstructor(nil) - if err != nil { - return nil, err - } - - // open the index - rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue) - err = rv.i.Open() - if err != nil { - return nil, err - } - rv.stats.indexStat = rv.i.Stats() - - // now persist the mapping - mappingBytes, err := json.Marshal(mapping) - if err != nil { - return nil, err - } - err = rv.i.SetInternal(mappingInternalKey, mappingBytes) - if err != nil { - return nil, err - } - - // mark the index as open - rv.mutex.Lock() - defer rv.mutex.Unlock() - rv.open = true - return &rv, nil -} - -func newIndexUsing(path string, mapping *IndexMapping, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) { +func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) { // first validate the mapping - err := mapping.validate() + err := mapping.Validate() if err != nil { return nil, err } - if path == "" { - return newMemIndex(mapping) - } - if kvconfig == nil { kvconfig = map[string]interface{}{} } - rv := indexImpl{ - path: path, - m: mapping, - meta: newIndexMeta(kvstore, kvconfig), - stats: &IndexStat{}, + if kvstore == "" { + return nil, fmt.Errorf("bleve not configured for file based indexing") } - storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) - if storeConstructor == nil { - return nil, ErrorUnknownStorageType - } - // at this point there is hope that we can be successful, so save index meta - err = rv.meta.Save(path) - if err != nil { - return nil, err - } - kvconfig["create_if_missing"] = true - kvconfig["error_if_exists"] = true - kvconfig["path"] = indexStorePath(path) - // now create the store - rv.s, err = storeConstructor(kvconfig) - if err != nil { - return nil, err + rv := indexImpl{ + path: path, + name: path, + m: mapping, + meta: newIndexMeta(indexType, kvstore, kvconfig), + } + rv.stats = &IndexStat{i: &rv} + // at this point there is hope that we can be successful, so save index meta + if path != "" { + err = rv.meta.Save(path) + if err != nil { + return nil, err + } + kvconfig["create_if_missing"] = true + kvconfig["error_if_exists"] = true + kvconfig["path"] = indexStorePath(path) + } else { + kvconfig["path"] = "" } // open the index - rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue) - err = rv.i.Open() + indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType) + if indexTypeConstructor == nil { + return nil, ErrorUnknownIndexType + } + + rv.i, err = indexTypeConstructor(rv.meta.Storage, kvconfig, Config.analysisQueue) if err != nil { return nil, err } - rv.stats.indexStat = rv.i.Stats() + err = rv.i.Open() + if err != nil { + if err == index.ErrorUnknownStorageType { + return nil, ErrorUnknownStorageType + } + return nil, err + } // now persist the mapping mappingBytes, err := json.Marshal(mapping) @@ -152,24 +122,25 @@ func newIndexUsing(path string, mapping *IndexMapping, kvstore string, kvconfig rv.mutex.Lock() defer rv.mutex.Unlock() rv.open = true + indexStats.Register(&rv) return &rv, nil } func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) { - rv = &indexImpl{ - path: path, - stats: &IndexStat{}, + path: path, + name: path, } + rv.stats = &IndexStat{i: rv} rv.meta, err = openIndexMeta(path) if err != nil { return nil, err } - storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage) - if storeConstructor == nil { - return nil, ErrorUnknownStorageType + // backwards compatibility if index type is missing + if rv.meta.IndexType == "" { + rv.meta.IndexType = upsidedown.Name } storeConfig := rv.meta.Config @@ -184,19 +155,23 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde storeConfig[rck] = rcv } - // now open the store - rv.s, err = storeConstructor(storeConfig) - if err != nil { - return nil, err + // open the index + indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType) + if indexTypeConstructor == nil { + return nil, ErrorUnknownIndexType } - // open the index - rv.i = upside_down.NewUpsideDownCouch(rv.s, Config.analysisQueue) - err = rv.i.Open() + rv.i, err = indexTypeConstructor(rv.meta.Storage, storeConfig, Config.analysisQueue) if err != nil { return nil, err } - rv.stats.indexStat = rv.i.Stats() + err = rv.i.Open() + if err != nil { + if err == index.ErrorUnknownStorageType { + return nil, ErrorUnknownStorageType + } + return nil, err + } // now load the mapping indexReader, err := rv.i.Reader() @@ -214,10 +189,10 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde return nil, err } - var im IndexMapping + var im *mapping.IndexMappingImpl err = json.Unmarshal(mappingBytes, &im) if err != nil { - return nil, err + return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes)) } // mark the index as open @@ -226,33 +201,42 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde rv.open = true // validate the mapping - err = im.validate() + err = im.Validate() if err != nil { // note even if the mapping is invalid // we still return an open usable index return rv, err } - rv.m = &im + rv.m = im + indexStats.Register(rv) return rv, err } // Advanced returns implementation internals // necessary ONLY for advanced usage. func (i *indexImpl) Advanced() (index.Index, store.KVStore, error) { - return i.i, i.s, nil + s, err := i.i.Advanced() + if err != nil { + return nil, nil, err + } + return i.i, s, nil } // Mapping returns the IndexMapping in use by this // Index. -func (i *indexImpl) Mapping() *IndexMapping { +func (i *indexImpl) Mapping() mapping.IndexMapping { return i.m } // Index the object with the specified identifier. // The IndexMapping for this index will determine // how the object is indexed. -func (i *indexImpl) Index(id string, data interface{}) error { +func (i *indexImpl) Index(id string, data interface{}) (err error) { + if id == "" { + return ErrorEmptyID + } + i.mutex.RLock() defer i.mutex.RUnlock() @@ -261,20 +245,21 @@ func (i *indexImpl) Index(id string, data interface{}) error { } doc := document.NewDocument(id) - err := i.m.mapDocument(doc, data) + err = i.m.MapDocument(doc, data) if err != nil { - return err + return } err = i.i.Update(doc) - if err != nil { - return err - } - return nil + return } // Delete entries for the specified identifier from // the index. -func (i *indexImpl) Delete(id string) error { +func (i *indexImpl) Delete(id string) (err error) { + if id == "" { + return ErrorEmptyID + } + i.mutex.RLock() defer i.mutex.RUnlock() @@ -282,11 +267,8 @@ func (i *indexImpl) Delete(id string) error { return ErrorIndexClosed } - err := i.i.Delete(id) - if err != nil { - return err - } - return nil + err = i.i.Delete(id) + return } // Batch executes multiple Index and Delete @@ -334,7 +316,7 @@ func (i *indexImpl) Document(id string) (doc *document.Document, err error) { // DocCount returns the number of documents in the // index. -func (i *indexImpl) DocCount() (uint64, error) { +func (i *indexImpl) DocCount() (count uint64, err error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -342,12 +324,30 @@ func (i *indexImpl) DocCount() (uint64, error) { return 0, ErrorIndexClosed } - return i.i.DocCount() + // open a reader for this search + indexReader, err := i.i.Reader() + if err != nil { + return 0, fmt.Errorf("error opening index reader %v", err) + } + defer func() { + if cerr := indexReader.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + count, err = indexReader.DocCount() + return } // Search executes a search request operation. // Returns a SearchResult object or an error. func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) { + return i.SearchInContext(context.Background(), req) +} + +// SearchInContext executes a search request operation within the provided +// Context. Returns a SearchResult object or an error. +func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr *SearchResult, err error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -357,7 +357,7 @@ func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) { return nil, ErrorIndexClosed } - collector := collectors.NewTopScorerSkipCollector(req.Size, req.From) + collector := collector.NewTopNCollector(req.Size, req.From, req.Sort) // open a reader for this search indexReader, err := i.i.Reader() @@ -370,7 +370,10 @@ func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) { } }() - searcher, err := req.Query.Searcher(indexReader, i.m, req.Explain) + searcher, err := req.Query.Searcher(indexReader, i.m, search.SearcherOptions{ + Explain: req.Explain, + IncludeTermVectors: req.IncludeLocations || req.Highlight != nil, + }) if err != nil { return nil, err } @@ -385,39 +388,41 @@ func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) { for facetName, facetRequest := range req.Facets { if facetRequest.NumericRanges != nil { // build numeric range facet - facetBuilder := facets.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size) + facetBuilder := facet.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size) for _, nr := range facetRequest.NumericRanges { facetBuilder.AddRange(nr.Name, nr.Min, nr.Max) } facetsBuilder.Add(facetName, facetBuilder) } else if facetRequest.DateTimeRanges != nil { // build date range facet - facetBuilder := facets.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size) - dateTimeParser := i.m.dateTimeParserNamed(i.m.DefaultDateTimeParser) + facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size) + dateTimeParser := i.m.DateTimeParserNamed("") for _, dr := range facetRequest.DateTimeRanges { - dr.ParseDates(dateTimeParser) - facetBuilder.AddRange(dr.Name, dr.Start, dr.End) + start, end := dr.ParseDates(dateTimeParser) + facetBuilder.AddRange(dr.Name, start, end) } facetsBuilder.Add(facetName, facetBuilder) } else { // build terms facet - facetBuilder := facets.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size) + facetBuilder := facet.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size) facetsBuilder.Add(facetName, facetBuilder) } } collector.SetFacetsBuilder(facetsBuilder) } - err = collector.Collect(searcher) + err = collector.Collect(ctx, searcher, indexReader) if err != nil { return nil, err } hits := collector.Results() + var highlighter highlight.Highlighter + if req.Highlight != nil { // get the right highlighter - highlighter, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter) + highlighter, err = Config.Cache.HighlighterNamed(Config.DefaultHighlighter) if err != nil { return nil, err } @@ -430,69 +435,91 @@ func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) { if highlighter == nil { return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style) } - - for _, hit := range hits { - doc, err := indexReader.Document(hit.ID) - if err == nil { - highlightFields := req.Highlight.Fields - if highlightFields == nil { - // add all fields with matches - highlightFields = make([]string, 0, len(hit.Locations)) - for k := range hit.Locations { - highlightFields = append(highlightFields, k) - } - } - - for _, hf := range highlightFields { - highlighter.BestFragmentsInField(hit, doc, hf, 1) - } - } - } } - if len(req.Fields) > 0 { - for _, hit := range hits { - // FIXME avoid loading doc second time - // if we already loaded it for highlighting + for _, hit := range hits { + if len(req.Fields) > 0 || highlighter != nil { doc, err := indexReader.Document(hit.ID) - if err == nil { - for _, f := range req.Fields { - for _, docF := range doc.Fields { - if f == "*" || docF.Name() == f { - var value interface{} - switch docF := docF.(type) { - case *document.TextField: - value = string(docF.Value()) - case *document.NumericField: - num, err := docF.Number() - if err == nil { - value = num + if err == nil && doc != nil { + if len(req.Fields) > 0 { + for _, f := range req.Fields { + for _, docF := range doc.Fields { + if f == "*" || docF.Name() == f { + var value interface{} + switch docF := docF.(type) { + case *document.TextField: + value = string(docF.Value()) + case *document.NumericField: + num, err := docF.Number() + if err == nil { + value = num + } + case *document.DateTimeField: + datetime, err := docF.DateTime() + if err == nil { + value = datetime.Format(time.RFC3339) + } + case *document.BooleanField: + boolean, err := docF.Boolean() + if err == nil { + value = boolean + } + case *document.GeoPointField: + lon, err := docF.Lon() + if err == nil { + lat, err := docF.Lat() + if err == nil { + value = []float64{lon, lat} + } + } } - case *document.DateTimeField: - datetime, err := docF.DateTime() - if err == nil { - value = datetime.Format(time.RFC3339) + if value != nil { + hit.AddFieldValue(docF.Name(), value) } } - if value != nil { - hit.AddFieldValue(docF.Name(), value) - } } } } + if highlighter != nil { + highlightFields := req.Highlight.Fields + if highlightFields == nil { + // add all fields with matches + highlightFields = make([]string, 0, len(hit.Locations)) + for k := range hit.Locations { + highlightFields = append(highlightFields, k) + } + } + for _, hf := range highlightFields { + highlighter.BestFragmentsInField(hit, doc, hf, 1) + } + } + } else if doc == nil { + // unexpected case, a doc ID that was found as a search hit + // was unable to be found during document lookup + return nil, ErrorIndexReadInconsistency } } + if i.name != "" { + hit.Index = i.name + } } atomic.AddUint64(&i.stats.searches, 1) searchDuration := time.Since(searchStart) atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration)) - if searchDuration > Config.SlowSearchLogThreshold { + if Config.SlowSearchLogThreshold > 0 && + searchDuration > Config.SlowSearchLogThreshold { logger.Printf("slow search took %s - %v", searchDuration, req) } return &SearchResult{ + Status: &SearchStatus{ + Total: 1, + Failed: 0, + Successful: 1, + Errors: make(map[string]error), + }, Request: req, Hits: hits, Total: collector.Total(), @@ -610,52 +637,12 @@ func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.Fiel }, nil } -// DumpAll writes all index rows to a channel. -// INTERNAL: do not rely on this function, it is -// only intended to be used by the debug utilities -func (i *indexImpl) DumpAll() chan interface{} { - i.mutex.RLock() - defer i.mutex.RUnlock() - - if !i.open { - return nil - } - - return i.i.DumpAll() -} - -// DumpFields writes all field rows in the index -// to a channel. -// INTERNAL: do not rely on this function, it is -// only intended to be used by the debug utilities -func (i *indexImpl) DumpFields() chan interface{} { - i.mutex.RLock() - defer i.mutex.RUnlock() - - if !i.open { - return nil - } - return i.i.DumpFields() -} - -// DumpDoc writes all rows in the index associated -// with the specified identifier to a channel. -// INTERNAL: do not rely on this function, it is -// only intended to be used by the debug utilities -func (i *indexImpl) DumpDoc(id string) chan interface{} { - i.mutex.RLock() - defer i.mutex.RUnlock() - - if !i.open { - return nil - } - return i.i.DumpDoc(id) -} - func (i *indexImpl) Close() error { i.mutex.Lock() defer i.mutex.Unlock() + indexStats.UnRegister(i) + i.open = false return i.i.Close() } @@ -664,6 +651,10 @@ func (i *indexImpl) Stats() *IndexStat { return i.stats } +func (i *indexImpl) StatsMap() map[string]interface{} { + return i.stats.statsMap() +} + func (i *indexImpl) GetInternal(key []byte) (val []byte, err error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -719,6 +710,16 @@ func (i *indexImpl) NewBatch() *Batch { } } +func (i *indexImpl) Name() string { + return i.name +} + +func (i *indexImpl) SetName(name string) { + indexStats.UnRegister(i) + i.name = name + indexStats.Register(i) +} + type indexImplFieldDict struct { index *indexImpl indexReader index.IndexReader diff --git a/vendor/github.com/blevesearch/bleve/index_meta.go b/vendor/github.com/blevesearch/bleve/index_meta.go index e59079b0..95592a65 100644 --- a/vendor/github.com/blevesearch/bleve/index_meta.go +++ b/vendor/github.com/blevesearch/bleve/index_meta.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve @@ -13,19 +18,23 @@ import ( "encoding/json" "io/ioutil" "os" + + "github.com/blevesearch/bleve/index/upsidedown" ) const metaFilename = "index_meta.json" type indexMeta struct { - Storage string `json:"storage"` - Config map[string]interface{} `json:"config,omitempty"` + Storage string `json:"storage"` + IndexType string `json:"index_type"` + Config map[string]interface{} `json:"config,omitempty"` } -func newIndexMeta(storage string, config map[string]interface{}) *indexMeta { +func newIndexMeta(indexType string, storage string, config map[string]interface{}) *indexMeta { return &indexMeta{ - Storage: storage, - Config: config, + IndexType: indexType, + Storage: storage, + Config: config, } } @@ -43,15 +52,21 @@ func openIndexMeta(path string) (*indexMeta, error) { if err != nil { return nil, ErrorIndexMetaCorrupt } + if im.IndexType == "" { + im.IndexType = upsidedown.Name + } return &im, nil } func (i *indexMeta) Save(path string) (err error) { indexMetaPath := indexMetaPath(path) // ensure any necessary parent directories exist - err = os.Mkdir(path, 0700) + err = os.MkdirAll(path, 0700) if err != nil { - return ErrorIndexPathExists + if os.IsExist(err) { + return ErrorIndexPathExists + } + return err } metaBytes, err := json.Marshal(i) if err != nil { diff --git a/vendor/github.com/blevesearch/bleve/index_meta_test.go b/vendor/github.com/blevesearch/bleve/index_meta_test.go deleted file mode 100644 index 4edb2a74..00000000 --- a/vendor/github.com/blevesearch/bleve/index_meta_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "os" - "testing" -) - -func TestIndexMeta(t *testing.T) { - var testIndexPath = "doesnotexit.bleve" - defer func() { - err := os.RemoveAll(testIndexPath) - if err != nil { - t.Fatal(err) - } - }() - - // open non-existent meta should give an error - _, err := openIndexMeta(testIndexPath) - if err == nil { - t.Errorf("expected error, got nil") - } - - // create meta - im := &indexMeta{Storage: "boltdb"} - err = im.Save(testIndexPath) - if err != nil { - t.Error(err) - } - im = nil - - // open a meta that exists - im, err = openIndexMeta(testIndexPath) - if err != nil { - t.Error(err) - } - if im.Storage != "boltdb" { - t.Errorf("expected storage 'boltdb', got '%s'", im.Storage) - } - - // save a meta that already exists - err = im.Save(testIndexPath) - if err == nil { - t.Errorf("expected error, got nil") - } -} diff --git a/vendor/github.com/blevesearch/bleve/index_stats.go b/vendor/github.com/blevesearch/bleve/index_stats.go index b29ec9ee..2d303f6e 100644 --- a/vendor/github.com/blevesearch/bleve/index_stats.go +++ b/vendor/github.com/blevesearch/bleve/index_stats.go @@ -1,39 +1,75 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve import ( "encoding/json" + "sync" "sync/atomic" ) type IndexStat struct { - indexStat json.Marshaler searches uint64 searchTime uint64 + i *indexImpl +} + +func (is *IndexStat) statsMap() map[string]interface{} { + m := map[string]interface{}{} + m["index"] = is.i.i.StatsMap() + m["searches"] = atomic.LoadUint64(&is.searches) + m["search_time"] = atomic.LoadUint64(&is.searchTime) + return m } func (is *IndexStat) MarshalJSON() ([]byte, error) { - m := map[string]interface{}{} - m["index"] = is.indexStat - m["searches"] = atomic.LoadUint64(&is.searches) - m["search_time"] = atomic.LoadUint64(&is.searchTime) + m := is.statsMap() return json.Marshal(m) } -type IndexStats map[string]*IndexStat +type IndexStats struct { + indexes map[string]*IndexStat + mutex sync.RWMutex +} -func (i IndexStats) String() string { - bytes, err := json.Marshal(i) +func NewIndexStats() *IndexStats { + return &IndexStats{ + indexes: make(map[string]*IndexStat), + } +} + +func (i *IndexStats) Register(index Index) { + i.mutex.Lock() + defer i.mutex.Unlock() + i.indexes[index.Name()] = index.Stats() +} + +func (i *IndexStats) UnRegister(index Index) { + i.mutex.Lock() + defer i.mutex.Unlock() + delete(i.indexes, index.Name()) +} + +func (i *IndexStats) String() string { + i.mutex.RLock() + defer i.mutex.RUnlock() + bytes, err := json.Marshal(i.indexes) if err != nil { return "error marshaling stats" } return string(bytes) } + +var indexStats *IndexStats diff --git a/vendor/github.com/blevesearch/bleve/index_test.go b/vendor/github.com/blevesearch/bleve/index_test.go deleted file mode 100644 index 8a0a7fe8..00000000 --- a/vendor/github.com/blevesearch/bleve/index_test.go +++ /dev/null @@ -1,609 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "io/ioutil" - "log" - "os" - "reflect" - "strings" - "testing" - "time" -) - -func TestCrud(t *testing.T) { - defer func() { - err := os.RemoveAll("testidx") - if err != nil { - t.Fatal(err) - } - }() - - index, err := New("testidx", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - - doca := map[string]interface{}{ - "name": "marty", - "desc": "gophercon india", - } - err = index.Index("a", doca) - if err != nil { - t.Error(err) - } - - docy := map[string]interface{}{ - "name": "jasper", - "desc": "clojure", - } - err = index.Index("y", docy) - if err != nil { - t.Error(err) - } - - err = index.Delete("y") - if err != nil { - t.Error(err) - } - - docx := map[string]interface{}{ - "name": "rose", - "desc": "googler", - } - err = index.Index("x", docx) - if err != nil { - t.Error(err) - } - - err = index.SetInternal([]byte("status"), []byte("pending")) - if err != nil { - t.Error(err) - } - - docb := map[string]interface{}{ - "name": "steve", - "desc": "cbft master", - } - batch := index.NewBatch() - err = batch.Index("b", docb) - if err != nil { - t.Error(err) - } - batch.Delete("x") - batch.SetInternal([]byte("batchi"), []byte("batchv")) - batch.DeleteInternal([]byte("status")) - err = index.Batch(batch) - if err != nil { - t.Error(err) - } - val, err := index.GetInternal([]byte("batchi")) - if err != nil { - t.Error(err) - } - if string(val) != "batchv" { - t.Errorf("expected 'batchv', got '%s'", val) - } - val, err = index.GetInternal([]byte("status")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got '%s'", val) - } - - err = index.SetInternal([]byte("seqno"), []byte("7")) - if err != nil { - t.Error(err) - } - err = index.SetInternal([]byte("status"), []byte("ready")) - if err != nil { - t.Error(err) - } - err = index.DeleteInternal([]byte("status")) - if err != nil { - t.Error(err) - } - val, err = index.GetInternal([]byte("status")) - if err != nil { - t.Error(err) - } - if val != nil { - t.Errorf("expected nil, got '%s'", val) - } - - val, err = index.GetInternal([]byte("seqno")) - if err != nil { - t.Error(err) - } - if string(val) != "7" { - t.Errorf("expected '7', got '%s'", val) - } - - // close the index, open it again, and try some more things - err = index.Close() - if err != nil { - t.Fatal(err) - } - - index, err = Open("testidx") - if err != nil { - t.Fatal(err) - } - defer func() { - err := index.Close() - if err != nil { - t.Fatal(err) - } - }() - - count, err := index.DocCount() - if err != nil { - t.Fatal(err) - } - if count != 2 { - t.Errorf("expected doc count 2, got %d", count) - } - - doc, err := index.Document("a") - if err != nil { - t.Fatal(err) - } - if doc == nil { - t.Errorf("expected doc not nil, got nil") - } - foundNameField := false - for _, field := range doc.Fields { - if field.Name() == "name" && string(field.Value()) == "marty" { - foundNameField = true - } - } - if !foundNameField { - t.Errorf("expected to find field named 'name' with value 'marty'") - } - - fields, err := index.Fields() - if err != nil { - t.Fatal(err) - } - expectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, - } - if len(fields) != len(expectedFields) { - t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) - } - for _, f := range fields { - expectedFields[f] = true - } - for ef, efp := range expectedFields { - if !efp { - t.Errorf("field %s is missing", ef) - } - } -} - -func TestIndexCreateNewOverExisting(t *testing.T) { - defer func() { - err := os.RemoveAll("testidx") - if err != nil { - t.Fatal(err) - } - }() - - index, err := New("testidx", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - err = index.Close() - if err != nil { - t.Fatal(err) - } - index, err = New("testidx", NewIndexMapping()) - if err != ErrorIndexPathExists { - t.Fatalf("expected error index path exists, got %v", err) - } -} - -func TestIndexOpenNonExisting(t *testing.T) { - _, err := Open("doesnotexist") - if err != ErrorIndexPathDoesNotExist { - t.Fatalf("expected error index path does not exist, got %v", err) - } -} - -func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) { - defer func() { - err := os.RemoveAll("testidx") - if err != nil { - t.Fatal(err) - } - }() - - index, err := New("testidx", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - err = index.Close() - if err != nil { - t.Fatal(err) - } - - // now intentionally change the storage type - err = ioutil.WriteFile("testidx/index_meta.json", []byte(`{"storage":"mystery"}`), 0666) - if err != nil { - t.Fatal(err) - } - - index, err = Open("testidx") - if err != ErrorUnknownStorageType { - t.Fatalf("expected error unknown storage type, got %v", err) - } - - // now intentionally corrupt the metadata - err = ioutil.WriteFile("testidx/index_meta.json", []byte("corrupted"), 0666) - if err != nil { - t.Fatal(err) - } - - index, err = Open("testidx") - if err != ErrorIndexMetaCorrupt { - t.Fatalf("expected error index metadata corrupted, got %v", err) - } - - // now intentionally remove the metadata - err = os.Remove("testidx/index_meta.json") - if err != nil { - t.Fatal(err) - } - - index, err = Open("testidx") - if err != ErrorIndexMetaMissing { - t.Fatalf("expected error index metadata missing, got %v", err) - } -} - -func TestInMemIndex(t *testing.T) { - - index, err := New("", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - err = index.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestClosedIndex(t *testing.T) { - index, err := New("", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - err = index.Close() - if err != nil { - t.Fatal(err) - } - - err = index.Index("test", "test") - if err != ErrorIndexClosed { - t.Errorf("expected error index closed, got %v", err) - } - - err = index.Delete("test") - if err != ErrorIndexClosed { - t.Errorf("expected error index closed, got %v", err) - } - - b := index.NewBatch() - err = index.Batch(b) - if err != ErrorIndexClosed { - t.Errorf("expected error index closed, got %v", err) - } - - _, err = index.Document("test") - if err != ErrorIndexClosed { - t.Errorf("expected error index closed, got %v", err) - } - - _, err = index.DocCount() - if err != ErrorIndexClosed { - t.Errorf("expected error index closed, got %v", err) - } - - _, err = index.Search(NewSearchRequest(NewTermQuery("test"))) - if err != ErrorIndexClosed { - t.Errorf("expected error index closed, got %v", err) - } - - _, err = index.Fields() - if err != ErrorIndexClosed { - t.Errorf("expected error index closed, got %v", err) - } -} - -func TestSlowSearch(t *testing.T) { - defer func() { - err := os.RemoveAll("testidx") - if err != nil { - t.Fatal(err) - } - }() - - defer func() { - // reset logger back to normal - SetLog(log.New(ioutil.Discard, "bleve", log.LstdFlags)) - }() - // set custom logger - var sdw sawDataWriter - SetLog(log.New(&sdw, "bleve", log.LstdFlags)) - - index, err := New("testidx", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - defer func() { - err := index.Close() - if err != nil { - t.Fatal(err) - } - }() - - Config.SlowSearchLogThreshold = 1 * time.Minute - - query := NewTermQuery("water") - req := NewSearchRequest(query) - _, err = index.Search(req) - if err != nil { - t.Fatal(err) - } - - if sdw.sawData { - t.Errorf("expected to not see slow query logged, but did") - } - - Config.SlowSearchLogThreshold = 1 * time.Microsecond - _, err = index.Search(req) - if err != nil { - t.Fatal(err) - } - - if !sdw.sawData { - t.Errorf("expected to see slow query logged, but didn't") - } -} - -type sawDataWriter struct { - sawData bool -} - -func (s *sawDataWriter) Write(p []byte) (n int, err error) { - s.sawData = true - return len(p), nil -} - -func TestStoredFieldPreserved(t *testing.T) { - defer func() { - err := os.RemoveAll("testidx") - if err != nil { - t.Fatal(err) - } - }() - - index, err := New("testidx", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - defer func() { - err := index.Close() - if err != nil { - t.Fatal(err) - } - }() - - doca := map[string]interface{}{ - "name": "Marty", - "desc": "GopherCON India", - } - err = index.Index("a", doca) - if err != nil { - t.Error(err) - } - - q := NewTermQuery("marty") - req := NewSearchRequest(q) - req.Fields = []string{"name", "desc"} - res, err := index.Search(req) - if err != nil { - t.Error(err) - } - - if len(res.Hits) != 1 { - t.Errorf("expected 1 hit, got %d", len(res.Hits)) - } - - if res.Hits[0].Fields["name"] != "Marty" { - t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["name"]) - } - if res.Hits[0].Fields["desc"] != "GopherCON India" { - t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["desc"]) - } - -} - -func TestDict(t *testing.T) { - defer func() { - err := os.RemoveAll("testidx") - if err != nil { - t.Fatal(err) - } - }() - - index, err := New("testidx", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - - doca := map[string]interface{}{ - "name": "marty", - "desc": "gophercon india", - } - err = index.Index("a", doca) - if err != nil { - t.Error(err) - } - - docy := map[string]interface{}{ - "name": "jasper", - "desc": "clojure", - } - err = index.Index("y", docy) - if err != nil { - t.Error(err) - } - - docx := map[string]interface{}{ - "name": "rose", - "desc": "googler", - } - err = index.Index("x", docx) - if err != nil { - t.Error(err) - } - - dict, err := index.FieldDict("name") - if err != nil { - t.Error(err) - } - - terms := []string{} - de, err := dict.Next() - for err == nil && de != nil { - terms = append(terms, string(de.Term)) - de, err = dict.Next() - } - - expectedTerms := []string{"jasper", "marty", "rose"} - if !reflect.DeepEqual(terms, expectedTerms) { - t.Errorf("expected %v, got %v", expectedTerms, terms) - } - - err = dict.Close() - if err != nil { - t.Fatal(err) - } - - // test start and end range - dict, err = index.FieldDictRange("name", []byte("marty"), []byte("rose")) - if err != nil { - t.Error(err) - } - - terms = []string{} - de, err = dict.Next() - for err == nil && de != nil { - terms = append(terms, string(de.Term)) - de, err = dict.Next() - } - - expectedTerms = []string{"marty", "rose"} - if !reflect.DeepEqual(terms, expectedTerms) { - t.Errorf("expected %v, got %v", expectedTerms, terms) - } - - err = dict.Close() - if err != nil { - t.Fatal(err) - } - - docz := map[string]interface{}{ - "name": "prefix", - "desc": "bob cat cats catting dog doggy zoo", - } - err = index.Index("z", docz) - if err != nil { - t.Error(err) - } - - dict, err = index.FieldDictPrefix("desc", []byte("cat")) - if err != nil { - t.Error(err) - } - - terms = []string{} - de, err = dict.Next() - for err == nil && de != nil { - terms = append(terms, string(de.Term)) - de, err = dict.Next() - } - - expectedTerms = []string{"cat", "cats", "catting"} - if !reflect.DeepEqual(terms, expectedTerms) { - t.Errorf("expected %v, got %v", expectedTerms, terms) - } - - stats := index.Stats() - if stats == nil { - t.Errorf("expected IndexStat, got nil") - } - - err = dict.Close() - if err != nil { - t.Fatal(err) - } - - err = index.Close() - if err != nil { - t.Fatal(err) - } -} - -func TestBatchString(t *testing.T) { - defer func() { - err := os.RemoveAll("testidx") - if err != nil { - t.Fatal(err) - } - }() - - index, err := New("testidx", NewIndexMapping()) - if err != nil { - t.Fatal(err) - } - - batch := index.NewBatch() - err = batch.Index("a", []byte("{}")) - if err != nil { - t.Fatal(err) - } - batch.Delete("b") - batch.SetInternal([]byte("c"), []byte{}) - batch.DeleteInternal([]byte("d")) - - batchStr := batch.String() - if !strings.HasPrefix(batchStr, "Batch (2 ops, 2 internal ops)") { - t.Errorf("expected to start with Batch (2 ops, 2 internal ops), did not") - } - if !strings.Contains(batchStr, "INDEX - 'a'") { - t.Errorf("expected to contain INDEX - 'a', did not") - } - if !strings.Contains(batchStr, "DELETE - 'b'") { - t.Errorf("expected to contain DELETE - 'b', did not") - } - if !strings.Contains(batchStr, "SET INTERNAL - 'c'") { - t.Errorf("expected to contain SET INTERNAL - 'c', did not") - } - if !strings.Contains(batchStr, "DELETE INTERNAL - 'd'") { - t.Errorf("expected to contain DELETE INTERNAL - 'd', did not") - } - -} diff --git a/vendor/github.com/blevesearch/bleve/mapping.go b/vendor/github.com/blevesearch/bleve/mapping.go new file mode 100644 index 00000000..76238dc1 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/mapping.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import "github.com/blevesearch/bleve/mapping" + +// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules +func NewIndexMapping() *mapping.IndexMappingImpl { + return mapping.NewIndexMapping() +} + +// NewDocumentMapping returns a new document mapping +// with all the default values. +func NewDocumentMapping() *mapping.DocumentMapping { + return mapping.NewDocumentMapping() +} + +// NewDocumentStaticMapping returns a new document +// mapping that will not automatically index parts +// of a document without an explicit mapping. +func NewDocumentStaticMapping() *mapping.DocumentMapping { + return mapping.NewDocumentStaticMapping() +} + +// NewDocumentDisabledMapping returns a new document +// mapping that will not perform any indexing. +func NewDocumentDisabledMapping() *mapping.DocumentMapping { + return mapping.NewDocumentDisabledMapping() +} + +// NewTextFieldMapping returns a default field mapping for text +func NewTextFieldMapping() *mapping.FieldMapping { + return mapping.NewTextFieldMapping() +} + +// NewNumericFieldMapping returns a default field mapping for numbers +func NewNumericFieldMapping() *mapping.FieldMapping { + return mapping.NewNumericFieldMapping() +} + +// NewDateTimeFieldMapping returns a default field mapping for dates +func NewDateTimeFieldMapping() *mapping.FieldMapping { + return mapping.NewDateTimeFieldMapping() +} + +// NewBooleanFieldMapping returns a default field mapping for booleans +func NewBooleanFieldMapping() *mapping.FieldMapping { + return mapping.NewBooleanFieldMapping() +} + +func NewGeoPointFieldMapping() *mapping.FieldMapping { + return mapping.NewGeoPointFieldMapping() +} diff --git a/vendor/github.com/blevesearch/bleve/mapping/analysis.go b/vendor/github.com/blevesearch/bleve/mapping/analysis.go new file mode 100644 index 00000000..03e3cd01 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/mapping/analysis.go @@ -0,0 +1,99 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +type customAnalysis struct { + CharFilters map[string]map[string]interface{} `json:"char_filters,omitempty"` + Tokenizers map[string]map[string]interface{} `json:"tokenizers,omitempty"` + TokenMaps map[string]map[string]interface{} `json:"token_maps,omitempty"` + TokenFilters map[string]map[string]interface{} `json:"token_filters,omitempty"` + Analyzers map[string]map[string]interface{} `json:"analyzers,omitempty"` + DateTimeParsers map[string]map[string]interface{} `json:"date_time_parsers,omitempty"` +} + +func (c *customAnalysis) registerAll(i *IndexMappingImpl) error { + for name, config := range c.CharFilters { + _, err := i.cache.DefineCharFilter(name, config) + if err != nil { + return err + } + } + + if len(c.Tokenizers) > 0 { + // put all the names in map tracking work to do + todo := map[string]struct{}{} + for name := range c.Tokenizers { + todo[name] = struct{}{} + } + registered := 1 + errs := []error{} + // as long as we keep making progress, keep going + for len(todo) > 0 && registered > 0 { + registered = 0 + errs = []error{} + for name := range todo { + config := c.Tokenizers[name] + _, err := i.cache.DefineTokenizer(name, config) + if err != nil { + errs = append(errs, err) + } else { + delete(todo, name) + registered++ + } + } + } + + if len(errs) > 0 { + return errs[0] + } + } + for name, config := range c.TokenMaps { + _, err := i.cache.DefineTokenMap(name, config) + if err != nil { + return err + } + } + for name, config := range c.TokenFilters { + _, err := i.cache.DefineTokenFilter(name, config) + if err != nil { + return err + } + } + for name, config := range c.Analyzers { + _, err := i.cache.DefineAnalyzer(name, config) + if err != nil { + return err + } + } + for name, config := range c.DateTimeParsers { + _, err := i.cache.DefineDateTimeParser(name, config) + if err != nil { + return err + } + } + return nil +} + +func newCustomAnalysis() *customAnalysis { + rv := customAnalysis{ + CharFilters: make(map[string]map[string]interface{}), + Tokenizers: make(map[string]map[string]interface{}), + TokenMaps: make(map[string]map[string]interface{}), + TokenFilters: make(map[string]map[string]interface{}), + Analyzers: make(map[string]map[string]interface{}), + DateTimeParsers: make(map[string]map[string]interface{}), + } + return &rv +} diff --git a/vendor/github.com/blevesearch/bleve/mapping_document.go b/vendor/github.com/blevesearch/bleve/mapping/document.go similarity index 52% rename from vendor/github.com/blevesearch/bleve/mapping_document.go rename to vendor/github.com/blevesearch/bleve/mapping/document.go index d0b3e2b4..9bdb8596 100644 --- a/vendor/github.com/blevesearch/bleve/mapping_document.go +++ b/vendor/github.com/blevesearch/bleve/mapping/document.go @@ -1,13 +1,18 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package bleve +package mapping import ( "encoding/json" @@ -37,9 +42,12 @@ type DocumentMapping struct { Properties map[string]*DocumentMapping `json:"properties,omitempty"` Fields []*FieldMapping `json:"fields,omitempty"` DefaultAnalyzer string `json:"default_analyzer"` + + // StructTagKey overrides "json" when looking for field names in struct tags + StructTagKey string `json:"struct_tag_key,omitempty"` } -func (dm *DocumentMapping) validate(cache *registry.Cache) error { +func (dm *DocumentMapping) Validate(cache *registry.Cache) error { var err error if dm.DefaultAnalyzer != "" { _, err := cache.AnalyzerNamed(dm.DefaultAnalyzer) @@ -48,7 +56,7 @@ func (dm *DocumentMapping) validate(cache *registry.Cache) error { } } for _, property := range dm.Properties { - err = property.validate(cache) + err = property.Validate(cache) if err != nil { return err } @@ -67,7 +75,7 @@ func (dm *DocumentMapping) validate(cache *registry.Cache) error { } } switch field.Type { - case "text", "datetime", "number": + case "text", "datetime", "number", "boolean", "geopoint": default: return fmt.Errorf("unknown field type: '%s'", field.Type) } @@ -75,54 +83,101 @@ func (dm *DocumentMapping) validate(cache *registry.Cache) error { return nil } +// analyzerNameForPath attempts to first find the field +// described by this path, then returns the analyzer +// configured for that field func (dm *DocumentMapping) analyzerNameForPath(path string) string { - pathElements := decodePath(path) - last := false - current := dm -OUTER: - for i, pathElement := range pathElements { - if i == len(pathElements)-1 { - last = true - } - for name, subDocMapping := range current.Properties { - for _, field := range subDocMapping.Fields { - if field.Name == "" && name == pathElement { - if last { - return field.Analyzer - } - current = subDocMapping - continue OUTER - } else if field.Name == pathElement { - if last { - return field.Analyzer - } - current = subDocMapping - continue OUTER - } - } - } - return "" + field := dm.fieldDescribedByPath(path) + if field != nil { + return field.Analyzer } return "" } +func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { + pathElements := decodePath(path) + if len(pathElements) > 1 { + // easy case, there is more than 1 path element remaining + // the next path element must match a property name + // at this level + for propName, subDocMapping := range dm.Properties { + if propName == pathElements[0] { + return subDocMapping.fieldDescribedByPath(encodePath(pathElements[1:])) + } + } + } else { + // just 1 path elememnt + // first look for property name with empty field + for propName, subDocMapping := range dm.Properties { + if propName == pathElements[0] { + // found property name match, now look at its fields + for _, field := range subDocMapping.Fields { + if field.Name == "" || field.Name == pathElements[0] { + // match + return field + } + } + } + } + // next, walk the properties again, looking for field overriding the name + for propName, subDocMapping := range dm.Properties { + if propName != pathElements[0] { + // property name isn't a match, but field name could override it + for _, field := range subDocMapping.Fields { + if field.Name == pathElements[0] { + return field + } + } + } + } + } + + return nil +} + +// documentMappingForPath only returns EXACT matches for a sub document +// or for an explicitly mapped field, if you want to find the +// closest document mapping to a field not explicitly mapped +// use closestDocMapping func (dm *DocumentMapping) documentMappingForPath(path string) *DocumentMapping { pathElements := decodePath(path) current := dm +OUTER: + for i, pathElement := range pathElements { + for name, subDocMapping := range current.Properties { + if name == pathElement { + current = subDocMapping + continue OUTER + } + } + // no subDocMapping matches this pathElement + // only if this is the last element check for field name + if i == len(pathElements)-1 { + for _, field := range current.Fields { + if field.Name == pathElement { + break + } + } + } + + return nil + } + return current +} + +// closestDocMapping findest the most specific document mapping that matches +// part of the provided path +func (dm *DocumentMapping) closestDocMapping(path string) *DocumentMapping { + pathElements := decodePath(path) + current := dm OUTER: for _, pathElement := range pathElements { for name, subDocMapping := range current.Properties { - for _, field := range subDocMapping.Fields { - if field.Name == "" && name == pathElement { - current = subDocMapping - continue OUTER - } else if field.Name == pathElement { - current = subDocMapping - continue OUTER - } + if name == pathElement { + current = subDocMapping + continue OUTER } } - return nil } return current } @@ -192,45 +247,61 @@ func (dm *DocumentMapping) AddFieldMapping(fm *FieldMapping) { dm.Fields = append(dm.Fields, fm) } -// UnmarshalJSON deserializes a JSON representation -// of the DocumentMapping. +// UnmarshalJSON offers custom unmarshaling with optional strict validation func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { - var tmp struct { - Enabled *bool `json:"enabled"` - Dynamic *bool `json:"dynamic"` - Properties map[string]*DocumentMapping `json:"properties"` - Fields []*FieldMapping `json:"fields"` - DefaultAnalyzer string `json:"default_analyzer"` - } + + var tmp map[string]json.RawMessage err := json.Unmarshal(data, &tmp) if err != nil { return err } + // set defaults for fields which might have been omitted dm.Enabled = true - if tmp.Enabled != nil { - dm.Enabled = *tmp.Enabled - } - dm.Dynamic = true - if tmp.Dynamic != nil { - dm.Dynamic = *tmp.Dynamic + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "enabled": + err := json.Unmarshal(v, &dm.Enabled) + if err != nil { + return err + } + case "dynamic": + err := json.Unmarshal(v, &dm.Dynamic) + if err != nil { + return err + } + case "default_analyzer": + err := json.Unmarshal(v, &dm.DefaultAnalyzer) + if err != nil { + return err + } + case "properties": + err := json.Unmarshal(v, &dm.Properties) + if err != nil { + return err + } + case "fields": + err := json.Unmarshal(v, &dm.Fields) + if err != nil { + return err + } + case "struct_tag_key": + err := json.Unmarshal(v, &dm.StructTagKey) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } } - dm.DefaultAnalyzer = tmp.DefaultAnalyzer + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("document mapping contains invalid keys: %v", invalidKeys) + } - if tmp.Properties != nil { - dm.Properties = make(map[string]*DocumentMapping, len(tmp.Properties)) - } - for propName, propMapping := range tmp.Properties { - dm.Properties[propName] = propMapping - } - if tmp.Fields != nil { - dm.Fields = make([]*FieldMapping, len(tmp.Fields)) - } - for i, field := range tmp.Fields { - dm.Fields[i] = field - } return nil } @@ -251,6 +322,12 @@ func (dm *DocumentMapping) defaultAnalyzerName(path []string) string { } func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) { + // allow default "json" tag to be overriden + structTagKey := dm.StructTagKey + if structTagKey == "" { + structTagKey = "json" + } + val := reflect.ValueOf(data) typ := val.Type() switch typ.Kind() { @@ -267,20 +344,29 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes for i := 0; i < val.NumField(); i++ { field := typ.Field(i) fieldName := field.Name + // anonymous fields of type struct can elide the type name + if field.Anonymous && field.Type.Kind() == reflect.Struct { + fieldName = "" + } - // if the field has a JSON name, prefer that - jsonTag := field.Tag.Get("json") - jsonFieldName := parseJSONTagName(jsonTag) - if jsonFieldName == "-" { + // if the field has a name under the specified tag, prefer that + tag := field.Tag.Get(structTagKey) + tagFieldName := parseTagName(tag) + if tagFieldName == "-" { continue } - if jsonFieldName != "" { - fieldName = jsonFieldName + // allow tag to set field name to empty, only if anonymous + if field.Tag != "" && (tagFieldName != "" || field.Anonymous) { + fieldName = tagFieldName } if val.Field(i).CanInterface() { fieldVal := val.Field(i).Interface() - dm.processProperty(fieldVal, append(path, fieldName), indexes, context) + newpath := path + if fieldName != "" { + newpath = append(path, fieldName) + } + dm.processProperty(fieldVal, newpath, indexes, context) } } case reflect.Slice, reflect.Array: @@ -295,13 +381,25 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes if ptrElem.IsValid() && ptrElem.CanInterface() { dm.processProperty(ptrElem.Interface(), path, indexes, context) } + case reflect.String: + dm.processProperty(val.String(), path, indexes, context) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + dm.processProperty(float64(val.Int()), path, indexes, context) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + dm.processProperty(float64(val.Uint()), path, indexes, context) + case reflect.Float32, reflect.Float64: + dm.processProperty(float64(val.Float()), path, indexes, context) + case reflect.Bool: + dm.processProperty(val.Bool(), path, indexes, context) } + } func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) { pathString := encodePath(path) // look to see if there is a mapping for this field subDocMapping := dm.documentMappingForPath(pathString) + closestDocMapping := dm.closestDocMapping(pathString) // check to see if we even need to do further processing if subDocMapping != nil && !subDocMapping.Enabled { @@ -322,36 +420,54 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, for _, fieldMapping := range subDocMapping.Fields { fieldMapping.processString(propertyValueString, pathString, path, indexes, context) } - } else { + } else if closestDocMapping.Dynamic { // automatic indexing behavior // first see if it can be parsed by the default date parser - dateTimeParser := context.im.dateTimeParserNamed(context.im.DefaultDateTimeParser) + dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) if dateTimeParser != nil { parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) if err != nil { // index as text - fieldMapping := NewTextFieldMapping() + fieldMapping := newTextFieldMappingDynamic(context.im) fieldMapping.processString(propertyValueString, pathString, path, indexes, context) } else { // index as datetime - fieldMapping := NewDateTimeFieldMapping() + fieldMapping := newDateTimeFieldMappingDynamic(context.im) fieldMapping.processTime(parsedDateTime, pathString, path, indexes, context) } } } - case reflect.Float64: + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + dm.processProperty(float64(propertyValue.Int()), path, indexes, context) + return + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + dm.processProperty(float64(propertyValue.Uint()), path, indexes, context) + return + case reflect.Float64, reflect.Float32: propertyValFloat := propertyValue.Float() if subDocMapping != nil { // index by explicit mapping for _, fieldMapping := range subDocMapping.Fields { fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) } - } else { + } else if closestDocMapping.Dynamic { // automatic indexing behavior - fieldMapping := NewNumericFieldMapping() + fieldMapping := newNumericFieldMappingDynamic(context.im) fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) } + case reflect.Bool: + propertyValBool := propertyValue.Bool() + if subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } + } else if closestDocMapping.Dynamic { + // automatic indexing behavior + fieldMapping := newBooleanFieldMappingDynamic(context.im) + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } case reflect.Struct: switch property := property.(type) { case time.Time: @@ -361,13 +477,29 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, for _, fieldMapping := range subDocMapping.Fields { fieldMapping.processTime(property, pathString, path, indexes, context) } - } else { - fieldMapping := NewDateTimeFieldMapping() + } else if closestDocMapping.Dynamic { + fieldMapping := newDateTimeFieldMappingDynamic(context.im) fieldMapping.processTime(property, pathString, path, indexes, context) } default: + if subDocMapping != nil { + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "geopoint" { + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + } + } + } dm.walkDocument(property, path, indexes, context) } + case reflect.Map: + if subDocMapping != nil { + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "geopoint" { + fieldMapping.processGeoPoint(property, pathString, path, indexes, context) + } + } + } + dm.walkDocument(property, path, indexes, context) default: dm.walkDocument(property, path, indexes, context) } diff --git a/vendor/github.com/blevesearch/bleve/mapping/field.go b/vendor/github.com/blevesearch/bleve/mapping/field.go new file mode 100644 index 00000000..9f1928ca --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/mapping/field.go @@ -0,0 +1,321 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/geo" +) + +// control the default behavior for dynamic fields (those not explicitly mapped) +var ( + IndexDynamic = true + StoreDynamic = true +) + +// A FieldMapping describes how a specific item +// should be put into the index. +type FieldMapping struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + + // Analyzer specifies the name of the analyzer to use for this field. If + // Analyzer is empty, traverse the DocumentMapping tree toward the root and + // pick the first non-empty DefaultAnalyzer found. If there is none, use + // the IndexMapping.DefaultAnalyzer. + Analyzer string `json:"analyzer,omitempty"` + + // Store indicates whether to store field values in the index. Stored + // values can be retrieved from search results using SearchRequest.Fields. + Store bool `json:"store,omitempty"` + Index bool `json:"index,omitempty"` + + // IncludeTermVectors, if true, makes terms occurrences to be recorded for + // this field. It includes the term position within the terms sequence and + // the term offsets in the source document field. Term vectors are required + // to perform phrase queries or terms highlighting in source documents. + IncludeTermVectors bool `json:"include_term_vectors,omitempty"` + IncludeInAll bool `json:"include_in_all,omitempty"` + DateFormat string `json:"date_format,omitempty"` +} + +// NewTextFieldMapping returns a default field mapping for text +func NewTextFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "text", + Store: true, + Index: true, + IncludeTermVectors: true, + IncludeInAll: true, + } +} + +func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewTextFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + return rv +} + +// NewNumericFieldMapping returns a default field mapping for numbers +func NewNumericFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "number", + Store: true, + Index: true, + IncludeInAll: true, + } +} + +func newNumericFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewNumericFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + return rv +} + +// NewDateTimeFieldMapping returns a default field mapping for dates +func NewDateTimeFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "datetime", + Store: true, + Index: true, + IncludeInAll: true, + } +} + +func newDateTimeFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewDateTimeFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + return rv +} + +// NewBooleanFieldMapping returns a default field mapping for booleans +func NewBooleanFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "boolean", + Store: true, + Index: true, + IncludeInAll: true, + } +} + +func newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { + rv := NewBooleanFieldMapping() + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + return rv +} + +// NewGeoPointFieldMapping returns a default field mapping for geo points +func NewGeoPointFieldMapping() *FieldMapping { + return &FieldMapping{ + Type: "geopoint", + Store: true, + Index: true, + IncludeInAll: true, + } +} + +// Options returns the indexing options for this field. +func (fm *FieldMapping) Options() document.IndexingOptions { + var rv document.IndexingOptions + if fm.Store { + rv |= document.StoreField + } + if fm.Index { + rv |= document.IndexField + } + if fm.IncludeTermVectors { + rv |= document.IncludeTermVectors + } + return rv +} + +func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + if fm.Type == "text" { + analyzer := fm.analyzerForField(path, context) + field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } else if fm.Type == "datetime" { + dateTimeFormat := context.im.DefaultDateTimeParser + if fm.DateFormat != "" { + dateTimeFormat = fm.DateFormat + } + dateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat) + if dateTimeParser != nil { + parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) + if err == nil { + fm.processTime(parsedDateTime, pathString, path, indexes, context) + } + } + } +} + +func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "number" { + options := fm.Options() + field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processTime(propertyValueTime time.Time, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "datetime" { + options := fm.Options() + field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, options) + if err == nil { + context.doc.AddField(field) + } else { + logger.Printf("could not build date %v", err) + } + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processBoolean(propertyValueBool bool, pathString string, path []string, indexes []uint64, context *walkContext) { + fieldName := getFieldName(pathString, path, fm) + if fm.Type == "boolean" { + options := fm.Options() + field := document.NewBooleanFieldWithIndexingOptions(fieldName, indexes, propertyValueBool, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) processGeoPoint(propertyMightBeGeoPoint interface{}, pathString string, path []string, indexes []uint64, context *walkContext) { + lon, lat, found := geo.ExtractGeoPoint(propertyMightBeGeoPoint) + if found { + fieldName := getFieldName(pathString, path, fm) + options := fm.Options() + field := document.NewGeoPointFieldWithIndexingOptions(fieldName, indexes, lon, lat, options) + context.doc.AddField(field) + + if !fm.IncludeInAll { + context.excludedFromAll = append(context.excludedFromAll, fieldName) + } + } +} + +func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) *analysis.Analyzer { + analyzerName := fm.Analyzer + if analyzerName == "" { + analyzerName = context.dm.defaultAnalyzerName(path) + if analyzerName == "" { + analyzerName = context.im.DefaultAnalyzer + } + } + return context.im.AnalyzerNamed(analyzerName) +} + +func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { + fieldName := pathString + if fieldMapping.Name != "" { + parentName := "" + if len(path) > 1 { + parentName = encodePath(path[:len(path)-1]) + pathSeparator + } + fieldName = parentName + fieldMapping.Name + } + return fieldName +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (fm *FieldMapping) UnmarshalJSON(data []byte) error { + + var tmp map[string]json.RawMessage + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "name": + err := json.Unmarshal(v, &fm.Name) + if err != nil { + return err + } + case "type": + err := json.Unmarshal(v, &fm.Type) + if err != nil { + return err + } + case "analyzer": + err := json.Unmarshal(v, &fm.Analyzer) + if err != nil { + return err + } + case "store": + err := json.Unmarshal(v, &fm.Store) + if err != nil { + return err + } + case "index": + err := json.Unmarshal(v, &fm.Index) + if err != nil { + return err + } + case "include_term_vectors": + err := json.Unmarshal(v, &fm.IncludeTermVectors) + if err != nil { + return err + } + case "include_in_all": + err := json.Unmarshal(v, &fm.IncludeInAll) + if err != nil { + return err + } + case "date_format": + err := json.Unmarshal(v, &fm.DateFormat) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("field mapping contains invalid keys: %v", invalidKeys) + } + + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/mapping/index.go b/vendor/github.com/blevesearch/bleve/mapping/index.go new file mode 100644 index 00000000..736e030b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/mapping/index.go @@ -0,0 +1,430 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/analysis/analyzer/standard" + "github.com/blevesearch/bleve/analysis/datetime/optional" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/registry" +) + +var MappingJSONStrict = false + +const defaultTypeField = "_type" +const defaultType = "_default" +const defaultField = "_all" +const defaultAnalyzer = standard.Name +const defaultDateTimeParser = optional.Name + +// An IndexMappingImpl controls how objects are placed +// into an index. +// First the type of the object is determined. +// Once the type is know, the appropriate +// DocumentMapping is selected by the type. +// If no mapping was determined for that type, +// a DefaultMapping will be used. +type IndexMappingImpl struct { + TypeMapping map[string]*DocumentMapping `json:"types,omitempty"` + DefaultMapping *DocumentMapping `json:"default_mapping"` + TypeField string `json:"type_field"` + DefaultType string `json:"default_type"` + DefaultAnalyzer string `json:"default_analyzer"` + DefaultDateTimeParser string `json:"default_datetime_parser"` + DefaultField string `json:"default_field"` + StoreDynamic bool `json:"store_dynamic"` + IndexDynamic bool `json:"index_dynamic"` + CustomAnalysis *customAnalysis `json:"analysis,omitempty"` + cache *registry.Cache +} + +// AddCustomCharFilter defines a custom char filter for use in this mapping +func (im *IndexMappingImpl) AddCustomCharFilter(name string, config map[string]interface{}) error { + _, err := im.cache.DefineCharFilter(name, config) + if err != nil { + return err + } + im.CustomAnalysis.CharFilters[name] = config + return nil +} + +// AddCustomTokenizer defines a custom tokenizer for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenizer(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenizer(name, config) + if err != nil { + return err + } + im.CustomAnalysis.Tokenizers[name] = config + return nil +} + +// AddCustomTokenMap defines a custom token map for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenMap(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenMap(name, config) + if err != nil { + return err + } + im.CustomAnalysis.TokenMaps[name] = config + return nil +} + +// AddCustomTokenFilter defines a custom token filter for use in this mapping +func (im *IndexMappingImpl) AddCustomTokenFilter(name string, config map[string]interface{}) error { + _, err := im.cache.DefineTokenFilter(name, config) + if err != nil { + return err + } + im.CustomAnalysis.TokenFilters[name] = config + return nil +} + +// AddCustomAnalyzer defines a custom analyzer for use in this mapping. The +// config map must have a "type" string entry to resolve the analyzer +// constructor. The constructor is invoked with the remaining entries and +// returned analyzer is registered in the IndexMapping. +// +// bleve comes with predefined analyzers, like +// github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer. They are +// available only if their package is imported by client code. To achieve this, +// use their metadata to fill configuration entries: +// +// import ( +// "github.com/blevesearch/bleve/analysis/analyzers/custom_analyzer" +// "github.com/blevesearch/bleve/analysis/char_filters/html_char_filter" +// "github.com/blevesearch/bleve/analysis/token_filters/lower_case_filter" +// "github.com/blevesearch/bleve/analysis/tokenizers/unicode" +// ) +// +// m := bleve.NewIndexMapping() +// err := m.AddCustomAnalyzer("html", map[string]interface{}{ +// "type": custom_analyzer.Name, +// "char_filters": []string{ +// html_char_filter.Name, +// }, +// "tokenizer": unicode.Name, +// "token_filters": []string{ +// lower_case_filter.Name, +// ... +// }, +// }) +func (im *IndexMappingImpl) AddCustomAnalyzer(name string, config map[string]interface{}) error { + _, err := im.cache.DefineAnalyzer(name, config) + if err != nil { + return err + } + im.CustomAnalysis.Analyzers[name] = config + return nil +} + +// AddCustomDateTimeParser defines a custom date time parser for use in this mapping +func (im *IndexMappingImpl) AddCustomDateTimeParser(name string, config map[string]interface{}) error { + _, err := im.cache.DefineDateTimeParser(name, config) + if err != nil { + return err + } + im.CustomAnalysis.DateTimeParsers[name] = config + return nil +} + +// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules +func NewIndexMapping() *IndexMappingImpl { + return &IndexMappingImpl{ + TypeMapping: make(map[string]*DocumentMapping), + DefaultMapping: NewDocumentMapping(), + TypeField: defaultTypeField, + DefaultType: defaultType, + DefaultAnalyzer: defaultAnalyzer, + DefaultDateTimeParser: defaultDateTimeParser, + DefaultField: defaultField, + IndexDynamic: IndexDynamic, + StoreDynamic: StoreDynamic, + CustomAnalysis: newCustomAnalysis(), + cache: registry.NewCache(), + } +} + +// Validate will walk the entire structure ensuring the following +// explicitly named and default analyzers can be built +func (im *IndexMappingImpl) Validate() error { + _, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer) + if err != nil { + return err + } + _, err = im.cache.DateTimeParserNamed(im.DefaultDateTimeParser) + if err != nil { + return err + } + err = im.DefaultMapping.Validate(im.cache) + if err != nil { + return err + } + for _, docMapping := range im.TypeMapping { + err = docMapping.Validate(im.cache) + if err != nil { + return err + } + } + return nil +} + +// AddDocumentMapping sets a custom document mapping for the specified type +func (im *IndexMappingImpl) AddDocumentMapping(doctype string, dm *DocumentMapping) { + im.TypeMapping[doctype] = dm +} + +func (im *IndexMappingImpl) mappingForType(docType string) *DocumentMapping { + docMapping := im.TypeMapping[docType] + if docMapping == nil { + docMapping = im.DefaultMapping + } + return docMapping +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { + + var tmp map[string]json.RawMessage + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + + // set defaults for fields which might have been omitted + im.cache = registry.NewCache() + im.CustomAnalysis = newCustomAnalysis() + im.TypeField = defaultTypeField + im.DefaultType = defaultType + im.DefaultAnalyzer = defaultAnalyzer + im.DefaultDateTimeParser = defaultDateTimeParser + im.DefaultField = defaultField + im.DefaultMapping = NewDocumentMapping() + im.TypeMapping = make(map[string]*DocumentMapping) + im.StoreDynamic = StoreDynamic + im.IndexDynamic = IndexDynamic + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "analysis": + err := json.Unmarshal(v, &im.CustomAnalysis) + if err != nil { + return err + } + case "type_field": + err := json.Unmarshal(v, &im.TypeField) + if err != nil { + return err + } + case "default_type": + err := json.Unmarshal(v, &im.DefaultType) + if err != nil { + return err + } + case "default_analyzer": + err := json.Unmarshal(v, &im.DefaultAnalyzer) + if err != nil { + return err + } + case "default_datetime_parser": + err := json.Unmarshal(v, &im.DefaultDateTimeParser) + if err != nil { + return err + } + case "default_field": + err := json.Unmarshal(v, &im.DefaultField) + if err != nil { + return err + } + case "default_mapping": + err := json.Unmarshal(v, &im.DefaultMapping) + if err != nil { + return err + } + case "types": + err := json.Unmarshal(v, &im.TypeMapping) + if err != nil { + return err + } + case "store_dynamic": + err := json.Unmarshal(v, &im.StoreDynamic) + if err != nil { + return err + } + case "index_dynamic": + err := json.Unmarshal(v, &im.IndexDynamic) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("index mapping contains invalid keys: %v", invalidKeys) + } + + err = im.CustomAnalysis.registerAll(im) + if err != nil { + return err + } + + return nil +} + +func (im *IndexMappingImpl) determineType(data interface{}) string { + // first see if the object implements Classifier + classifier, ok := data.(Classifier) + if ok { + return classifier.Type() + } + + // now see if we can find a type using the mapping + typ, ok := mustString(lookupPropertyPath(data, im.TypeField)) + if ok { + return typ + } + + return im.DefaultType +} + +func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}) error { + docType := im.determineType(data) + docMapping := im.mappingForType(docType) + walkContext := im.newWalkContext(doc, docMapping) + if docMapping.Enabled { + docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) + + // see if the _all field was disabled + allMapping := docMapping.documentMappingForPath("_all") + if allMapping == nil || (allMapping.Enabled != false) { + field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) + doc.AddField(field) + } + } + + return nil +} + +type walkContext struct { + doc *document.Document + im *IndexMappingImpl + dm *DocumentMapping + excludedFromAll []string +} + +func (im *IndexMappingImpl) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext { + return &walkContext{ + doc: doc, + im: im, + dm: dm, + excludedFromAll: []string{}, + } +} + +// AnalyzerNameForPath attempts to find the best analyzer to use with only a +// field name will walk all the document types, look for field mappings at the +// provided path, if one exists and it has an explicit analyzer that is +// returned. +func (im *IndexMappingImpl) AnalyzerNameForPath(path string) string { + // first we look for explicit mapping on the field + for _, docMapping := range im.TypeMapping { + analyzerName := docMapping.analyzerNameForPath(path) + if analyzerName != "" { + return analyzerName + } + } + // now try the default mapping + pathMapping := im.DefaultMapping.documentMappingForPath(path) + if pathMapping != nil { + if len(pathMapping.Fields) > 0 { + if pathMapping.Fields[0].Analyzer != "" { + return pathMapping.Fields[0].Analyzer + } + } + } + + // next we will try default analyzers for the path + pathDecoded := decodePath(path) + for _, docMapping := range im.TypeMapping { + rv := docMapping.defaultAnalyzerName(pathDecoded) + if rv != "" { + return rv + } + } + + return im.DefaultAnalyzer +} + +func (im *IndexMappingImpl) AnalyzerNamed(name string) *analysis.Analyzer { + analyzer, err := im.cache.AnalyzerNamed(name) + if err != nil { + logger.Printf("error using analyzer named: %s", name) + return nil + } + return analyzer +} + +func (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimeParser { + if name == "" { + name = im.DefaultDateTimeParser + } + dateTimeParser, err := im.cache.DateTimeParserNamed(name) + if err != nil { + logger.Printf("error using datetime parser named: %s", name) + return nil + } + return dateTimeParser +} + +func (im *IndexMappingImpl) datetimeParserNameForPath(path string) string { + + // first we look for explicit mapping on the field + for _, docMapping := range im.TypeMapping { + pathMapping := docMapping.documentMappingForPath(path) + if pathMapping != nil { + if len(pathMapping.Fields) > 0 { + if pathMapping.Fields[0].Analyzer != "" { + return pathMapping.Fields[0].Analyzer + } + } + } + } + + return im.DefaultDateTimeParser +} + +func (im *IndexMappingImpl) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) { + analyzer, err := im.cache.AnalyzerNamed(analyzerName) + if err != nil { + return nil, err + } + return analyzer.Analyze(text), nil +} + +// FieldAnalyzer returns the name of the analyzer used on a field. +func (im *IndexMappingImpl) FieldAnalyzer(field string) string { + return im.AnalyzerNameForPath(field) +} + +// wrapper to satisfy new interface + +func (im *IndexMappingImpl) DefaultSearchField() string { + return im.DefaultField +} diff --git a/vendor/github.com/blevesearch/bleve/mapping/mapping.go b/vendor/github.com/blevesearch/bleve/mapping/mapping.go new file mode 100644 index 00000000..61002110 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/mapping/mapping.go @@ -0,0 +1,49 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "io/ioutil" + "log" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/document" +) + +// A Classifier is an interface describing any object +// which knows how to identify its own type. +type Classifier interface { + Type() string +} + +var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to ioutil.Discard +func SetLog(l *log.Logger) { + logger = l +} + +type IndexMapping interface { + MapDocument(doc *document.Document, data interface{}) error + Validate() error + + DateTimeParserNamed(name string) analysis.DateTimeParser + + DefaultSearchField() string + + AnalyzerNameForPath(path string) string + AnalyzerNamed(name string) *analysis.Analyzer +} diff --git a/vendor/github.com/blevesearch/bleve/reflect.go b/vendor/github.com/blevesearch/bleve/mapping/reflect.go similarity index 55% rename from vendor/github.com/blevesearch/bleve/reflect.go rename to vendor/github.com/blevesearch/bleve/mapping/reflect.go index f299c912..3068b190 100644 --- a/vendor/github.com/blevesearch/bleve/reflect.go +++ b/vendor/github.com/blevesearch/bleve/mapping/reflect.go @@ -1,13 +1,18 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package bleve +package mapping import ( "reflect" @@ -19,7 +24,7 @@ func lookupPropertyPath(data interface{}, path string) interface{} { current := data for _, part := range pathParts { - current = lookupProptyPathPart(current, part) + current = lookupPropertyPathPart(current, part) if current == nil { break } @@ -28,7 +33,7 @@ func lookupPropertyPath(data interface{}, path string) interface{} { return current } -func lookupProptyPathPart(data interface{}, part string) interface{} { +func lookupPropertyPathPart(data interface{}, part string) interface{} { val := reflect.ValueOf(data) typ := val.Type() switch typ.Kind() { @@ -46,6 +51,11 @@ func lookupProptyPathPart(data interface{}, part string) interface{} { if field.IsValid() && field.CanInterface() { return field.Interface() } + case reflect.Ptr: + ptrElem := val.Elem() + if ptrElem.IsValid() && ptrElem.CanInterface() { + return lookupPropertyPathPart(ptrElem.Interface(), part) + } } return nil } @@ -70,8 +80,8 @@ func mustString(data interface{}) (string, bool) { return "", false } -// parseJSONTagName extracts the JSON field name from a struct tag -func parseJSONTagName(tag string) string { +// parseTagName extracts the field name from a struct tag +func parseTagName(tag string) string { if idx := strings.Index(tag, ","); idx != -1 { return tag[:idx] } diff --git a/vendor/github.com/blevesearch/bleve/mapping_field.go b/vendor/github.com/blevesearch/bleve/mapping_field.go deleted file mode 100644 index 3ff4461b..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping_field.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "time" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" -) - -// A FieldMapping describes how a specific item -// should be put into the index. -type FieldMapping struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Analyzer string `json:"analyzer,omitempty"` - Store bool `json:"store,omitempty"` - Index bool `json:"index,omitempty"` - IncludeTermVectors bool `json:"include_term_vectors,omitempty"` - IncludeInAll bool `json:"include_in_all,omitempty"` - DateFormat string `json:"date_format,omitempty"` -} - -// NewTextFieldMapping returns a default field mapping for text -func NewTextFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "text", - Store: true, - Index: true, - IncludeTermVectors: true, - IncludeInAll: true, - } -} - -// NewNumericFieldMapping returns a default field mapping for numbers -func NewNumericFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "number", - Store: true, - Index: true, - IncludeInAll: true, - } -} - -// NewDateTimeFieldMapping returns a default field mapping for dates -func NewDateTimeFieldMapping() *FieldMapping { - return &FieldMapping{ - Type: "datetime", - Store: true, - Index: true, - IncludeInAll: true, - } -} - -// Options returns the indexing options for this field. -func (fm *FieldMapping) Options() document.IndexingOptions { - var rv document.IndexingOptions - if fm.Store { - rv |= document.StoreField - } - if fm.Index { - rv |= document.IndexField - } - if fm.IncludeTermVectors { - rv |= document.IncludeTermVectors - } - return rv -} - -func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) { - fieldName := getFieldName(pathString, path, fm) - options := fm.Options() - if fm.Type == "text" { - analyzer := fm.analyzerForField(path, context) - field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer) - context.doc.AddField(field) - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } else if fm.Type == "datetime" { - dateTimeFormat := context.im.DefaultDateTimeParser - if fm.DateFormat != "" { - dateTimeFormat = fm.DateFormat - } - dateTimeParser := context.im.dateTimeParserNamed(dateTimeFormat) - if dateTimeParser != nil { - parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) - if err != nil { - fm.processTime(parsedDateTime, pathString, path, indexes, context) - } - } - } -} - -func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) { - fieldName := getFieldName(pathString, path, fm) - if fm.Type == "number" { - options := fm.Options() - field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options) - context.doc.AddField(field) - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } -} - -func (fm *FieldMapping) processTime(propertyValueTime time.Time, pathString string, path []string, indexes []uint64, context *walkContext) { - fieldName := getFieldName(pathString, path, fm) - if fm.Type == "datetime" { - options := fm.Options() - field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, options) - if err == nil { - context.doc.AddField(field) - } else { - logger.Printf("could not build date %v", err) - } - - if !fm.IncludeInAll { - context.excludedFromAll = append(context.excludedFromAll, fieldName) - } - } -} - -func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) *analysis.Analyzer { - analyzerName := context.dm.defaultAnalyzerName(path) - if analyzerName == "" { - analyzerName = context.im.DefaultAnalyzer - } - if fm.Analyzer != "" { - analyzerName = fm.Analyzer - } - return context.im.analyzerNamed(analyzerName) -} - -func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { - fieldName := pathString - if fieldMapping.Name != "" { - parentName := "" - if len(path) > 1 { - parentName = encodePath(path[:len(path)-1]) + pathSeparator - } - fieldName = parentName + fieldMapping.Name - } - return fieldName -} diff --git a/vendor/github.com/blevesearch/bleve/mapping_index.go b/vendor/github.com/blevesearch/bleve/mapping_index.go deleted file mode 100644 index edb95f49..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping_index.go +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "encoding/json" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/registry" -) - -const defaultTypeField = "_type" -const defaultType = "_default" -const defaultField = "_all" -const defaultAnalyzer = "standard" -const defaultDateTimeParser = "dateTimeOptional" -const defaultByteArrayConverter = "json" - -type customAnalysis struct { - CharFilters map[string]map[string]interface{} `json:"char_filters,omitempty"` - Tokenizers map[string]map[string]interface{} `json:"tokenizers,omitempty"` - TokenMaps map[string]map[string]interface{} `json:"token_maps,omitempty"` - TokenFilters map[string]map[string]interface{} `json:"token_filters,omitempty"` - Analyzers map[string]map[string]interface{} `json:"analyzers,omitempty"` - DateTimeParsers map[string]map[string]interface{} `json:"date_time_parsers,omitempty"` -} - -func (c *customAnalysis) registerAll(i *IndexMapping) error { - for name, config := range c.CharFilters { - _, err := i.cache.DefineCharFilter(name, config) - if err != nil { - return err - } - } - for name, config := range c.Tokenizers { - _, err := i.cache.DefineTokenizer(name, config) - if err != nil { - return err - } - } - for name, config := range c.TokenMaps { - _, err := i.cache.DefineTokenMap(name, config) - if err != nil { - return err - } - } - for name, config := range c.TokenFilters { - _, err := i.cache.DefineTokenFilter(name, config) - if err != nil { - return err - } - } - for name, config := range c.Analyzers { - _, err := i.cache.DefineAnalyzer(name, config) - if err != nil { - return err - } - } - for name, config := range c.DateTimeParsers { - _, err := i.cache.DefineDateTimeParser(name, config) - if err != nil { - return err - } - } - return nil -} - -func newCustomAnalysis() *customAnalysis { - rv := customAnalysis{ - CharFilters: make(map[string]map[string]interface{}), - Tokenizers: make(map[string]map[string]interface{}), - TokenMaps: make(map[string]map[string]interface{}), - TokenFilters: make(map[string]map[string]interface{}), - Analyzers: make(map[string]map[string]interface{}), - DateTimeParsers: make(map[string]map[string]interface{}), - } - return &rv -} - -// An IndexMapping controls how objects are placed -// into an index. -// First the type of the object is determined. -// Once the type is know, the appropriate -// DocumentMapping is selected by the type. -// If no mapping was determined for that type, -// a DefaultMapping will be used. -type IndexMapping struct { - TypeMapping map[string]*DocumentMapping `json:"types,omitempty"` - DefaultMapping *DocumentMapping `json:"default_mapping"` - TypeField string `json:"type_field"` - DefaultType string `json:"default_type"` - DefaultAnalyzer string `json:"default_analyzer"` - DefaultDateTimeParser string `json:"default_datetime_parser"` - DefaultField string `json:"default_field"` - ByteArrayConverter string `json:"byte_array_converter"` - CustomAnalysis *customAnalysis `json:"analysis,omitempty"` - cache *registry.Cache -} - -// AddCustomCharFilter defines a custom char filter for use in this mapping -func (im *IndexMapping) AddCustomCharFilter(name string, config map[string]interface{}) error { - _, err := im.cache.DefineCharFilter(name, config) - if err != nil { - return err - } - im.CustomAnalysis.CharFilters[name] = config - return nil -} - -// AddCustomTokenizer defines a custom tokenizer for use in this mapping -func (im *IndexMapping) AddCustomTokenizer(name string, config map[string]interface{}) error { - _, err := im.cache.DefineTokenizer(name, config) - if err != nil { - return err - } - im.CustomAnalysis.Tokenizers[name] = config - return nil -} - -// AddCustomTokenMap defines a custom token map for use in this mapping -func (im *IndexMapping) AddCustomTokenMap(name string, config map[string]interface{}) error { - _, err := im.cache.DefineTokenMap(name, config) - if err != nil { - return err - } - im.CustomAnalysis.TokenMaps[name] = config - return nil -} - -// AddCustomTokenFilter defines a custom token filter for use in this mapping -func (im *IndexMapping) AddCustomTokenFilter(name string, config map[string]interface{}) error { - _, err := im.cache.DefineTokenFilter(name, config) - if err != nil { - return err - } - im.CustomAnalysis.TokenFilters[name] = config - return nil -} - -// AddCustomAnalyzer defines a custom analyzer for use in this mapping -func (im *IndexMapping) AddCustomAnalyzer(name string, config map[string]interface{}) error { - _, err := im.cache.DefineAnalyzer(name, config) - if err != nil { - return err - } - im.CustomAnalysis.Analyzers[name] = config - return nil -} - -// AddCustomDateTimeParser defines a custom date time parser for use in this mapping -func (im *IndexMapping) AddCustomDateTimeParser(name string, config map[string]interface{}) error { - _, err := im.cache.DefineDateTimeParser(name, config) - if err != nil { - return err - } - im.CustomAnalysis.DateTimeParsers[name] = config - return nil -} - -// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules -func NewIndexMapping() *IndexMapping { - return &IndexMapping{ - TypeMapping: make(map[string]*DocumentMapping), - DefaultMapping: NewDocumentMapping(), - TypeField: defaultTypeField, - DefaultType: defaultType, - DefaultAnalyzer: defaultAnalyzer, - DefaultDateTimeParser: defaultDateTimeParser, - DefaultField: defaultField, - ByteArrayConverter: defaultByteArrayConverter, - CustomAnalysis: newCustomAnalysis(), - cache: registry.NewCache(), - } -} - -// Validate will walk the entire structure ensuring the following -// explicitly named and default analyzers can be built -func (im *IndexMapping) validate() error { - _, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer) - if err != nil { - return err - } - _, err = im.cache.DateTimeParserNamed(im.DefaultDateTimeParser) - if err != nil { - return err - } - err = im.DefaultMapping.validate(im.cache) - if err != nil { - return err - } - for _, docMapping := range im.TypeMapping { - err = docMapping.validate(im.cache) - if err != nil { - return err - } - } - return nil -} - -// AddDocumentMapping sets a custom document mapping for the specified type -func (im *IndexMapping) AddDocumentMapping(doctype string, dm *DocumentMapping) { - im.TypeMapping[doctype] = dm -} - -func (im *IndexMapping) mappingForType(docType string) *DocumentMapping { - docMapping := im.TypeMapping[docType] - if docMapping == nil { - docMapping = im.DefaultMapping - } - return docMapping -} - -// UnmarshalJSON deserializes a JSON representation of the IndexMapping -func (im *IndexMapping) UnmarshalJSON(data []byte) error { - var tmp struct { - TypeMapping map[string]*DocumentMapping `json:"types"` - DefaultMapping *DocumentMapping `json:"default_mapping"` - TypeField string `json:"type_field"` - DefaultType string `json:"default_type"` - DefaultAnalyzer string `json:"default_analyzer"` - DefaultDateTimeParser string `json:"default_datetime_parser"` - DefaultField string `json:"default_field"` - ByteArrayConverter string `json:"byte_array_converter"` - CustomAnalysis *customAnalysis `json:"analysis"` - } - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - - im.cache = registry.NewCache() - - im.CustomAnalysis = newCustomAnalysis() - if tmp.CustomAnalysis != nil { - if tmp.CustomAnalysis.CharFilters != nil { - im.CustomAnalysis.CharFilters = tmp.CustomAnalysis.CharFilters - } - if tmp.CustomAnalysis.Tokenizers != nil { - im.CustomAnalysis.Tokenizers = tmp.CustomAnalysis.Tokenizers - } - if tmp.CustomAnalysis.TokenMaps != nil { - im.CustomAnalysis.TokenMaps = tmp.CustomAnalysis.TokenMaps - } - if tmp.CustomAnalysis.TokenFilters != nil { - im.CustomAnalysis.TokenFilters = tmp.CustomAnalysis.TokenFilters - } - if tmp.CustomAnalysis.Analyzers != nil { - im.CustomAnalysis.Analyzers = tmp.CustomAnalysis.Analyzers - } - if tmp.CustomAnalysis.DateTimeParsers != nil { - im.CustomAnalysis.DateTimeParsers = tmp.CustomAnalysis.DateTimeParsers - } - } - - im.TypeField = defaultTypeField - if tmp.TypeField != "" { - im.TypeField = tmp.TypeField - } - - im.DefaultType = defaultType - if tmp.DefaultType != "" { - im.DefaultType = tmp.DefaultType - } - - im.DefaultAnalyzer = defaultAnalyzer - if tmp.DefaultAnalyzer != "" { - im.DefaultAnalyzer = tmp.DefaultAnalyzer - } - - im.DefaultDateTimeParser = defaultDateTimeParser - if tmp.DefaultDateTimeParser != "" { - im.DefaultDateTimeParser = tmp.DefaultDateTimeParser - } - - im.DefaultField = defaultField - if tmp.DefaultField != "" { - im.DefaultField = tmp.DefaultField - } - - im.ByteArrayConverter = defaultByteArrayConverter - if tmp.ByteArrayConverter != "" { - im.ByteArrayConverter = tmp.ByteArrayConverter - } - - im.DefaultMapping = NewDocumentMapping() - if tmp.DefaultMapping != nil { - im.DefaultMapping = tmp.DefaultMapping - } - - im.TypeMapping = make(map[string]*DocumentMapping, len(tmp.TypeMapping)) - for typeName, typeDocMapping := range tmp.TypeMapping { - im.TypeMapping[typeName] = typeDocMapping - } - - err = im.CustomAnalysis.registerAll(im) - if err != nil { - return err - } - - return nil -} - -func (im *IndexMapping) determineType(data interface{}) string { - // first see if the object implements Identifier - classifier, ok := data.(Classifier) - if ok { - return classifier.Type() - } - - // now see if we can find a type using the mapping - typ, ok := mustString(lookupPropertyPath(data, im.TypeField)) - if ok { - return typ - } - - return im.DefaultType -} - -func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) error { - // see if the top level object is a byte array, and possibly run through a converter - byteArrayData, ok := data.([]byte) - if ok { - byteArrayConverterConstructor := registry.ByteArrayConverterByName(im.ByteArrayConverter) - if byteArrayConverterConstructor != nil { - byteArrayConverter, err := byteArrayConverterConstructor(nil, nil) - if err == nil { - convertedData, err := byteArrayConverter.Convert(byteArrayData) - if err != nil { - return err - } - data = convertedData - } else { - logger.Printf("error creating byte array converter: %v", err) - } - } else { - logger.Printf("no byte array converter named: %s", im.ByteArrayConverter) - } - } - - docType := im.determineType(data) - docMapping := im.mappingForType(docType) - walkContext := im.newWalkContext(doc, docMapping) - docMapping.walkDocument(data, []string{}, []uint64{}, walkContext) - - // see if the _all field was disabled - allMapping := docMapping.documentMappingForPath("_all") - if allMapping == nil || (allMapping.Enabled != false) { - field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) - doc.AddField(field) - } - - return nil -} - -type walkContext struct { - doc *document.Document - im *IndexMapping - dm *DocumentMapping - excludedFromAll []string -} - -func (im *IndexMapping) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext { - return &walkContext{ - doc: doc, - im: im, - dm: dm, - excludedFromAll: []string{}, - } -} - -// attempts to find the best analyzer to use with only a field name -// will walk all the document types, look for field mappings at the -// provided path, if one exists and it has an explicit analyzer -// that is returned -// nil should be an acceptable return value meaning we don't know -func (im *IndexMapping) analyzerNameForPath(path string) string { - // first we look for explicit mapping on the field - for _, docMapping := range im.TypeMapping { - analyzerName := docMapping.analyzerNameForPath(path) - if analyzerName != "" { - return analyzerName - } - } - // now try the default mapping - pathMapping := im.DefaultMapping.documentMappingForPath(path) - if pathMapping != nil { - if len(pathMapping.Fields) > 0 { - if pathMapping.Fields[0].Analyzer != "" { - return pathMapping.Fields[0].Analyzer - } - } - } - - // next we will try default analyzers for the path - pathDecoded := decodePath(path) - for _, docMapping := range im.TypeMapping { - rv := docMapping.defaultAnalyzerName(pathDecoded) - if rv != "" { - return rv - } - } - - return im.DefaultAnalyzer -} - -func (im *IndexMapping) analyzerNamed(name string) *analysis.Analyzer { - analyzer, err := im.cache.AnalyzerNamed(name) - if err != nil { - logger.Printf("error using analyzer named: %s", name) - return nil - } - return analyzer -} - -func (im *IndexMapping) dateTimeParserNamed(name string) analysis.DateTimeParser { - dateTimeParser, err := im.cache.DateTimeParserNamed(name) - if err != nil { - logger.Printf("error using datetime parser named: %s", name) - return nil - } - return dateTimeParser -} - -func (im *IndexMapping) datetimeParserNameForPath(path string) string { - - // first we look for explicit mapping on the field - for _, docMapping := range im.TypeMapping { - pathMapping := docMapping.documentMappingForPath(path) - if pathMapping != nil { - if len(pathMapping.Fields) > 0 { - if pathMapping.Fields[0].Analyzer != "" { - return pathMapping.Fields[0].Analyzer - } - } - } - } - - return im.DefaultDateTimeParser -} - -func (im *IndexMapping) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) { - analyzer, err := im.cache.AnalyzerNamed(analyzerName) - if err != nil { - return nil, err - } - return analyzer.Analyze(text), nil -} diff --git a/vendor/github.com/blevesearch/bleve/mapping_test.go b/vendor/github.com/blevesearch/bleve/mapping_test.go deleted file mode 100644 index 6d947768..00000000 --- a/vendor/github.com/blevesearch/bleve/mapping_test.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "encoding/json" - "github.com/blevesearch/bleve/document" - "reflect" - "testing" -) - -var mappingSource = []byte(`{ - "types": { - "beer": { - "properties": { - "name": { - "fields": [ - { - "name": "name", - "type": "text", - "analyzer": "standard", - "store": true, - "index": true, - "include_term_vectors": true, - "include_in_all": true - } - ] - } - } - }, - "brewery": { - } - }, - "type_field": "_type", - "default_type": "_default" -}`) - -func buildMapping() *IndexMapping { - nameFieldMapping := NewTextFieldMapping() - nameFieldMapping.Name = "name" - nameFieldMapping.Analyzer = "standard" - - beerMapping := NewDocumentMapping() - beerMapping.AddFieldMappingsAt("name", nameFieldMapping) - - breweryMapping := NewDocumentMapping() - - mapping := NewIndexMapping() - mapping.AddDocumentMapping("beer", beerMapping) - mapping.AddDocumentMapping("brewery", breweryMapping) - - return mapping -} - -func TestUnmarshalMappingJSON(t *testing.T) { - mapping := buildMapping() - - var indexMapping IndexMapping - err := json.Unmarshal(mappingSource, &indexMapping) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(&indexMapping, mapping) { - t.Errorf("expected %#v,\n got %#v", mapping, &indexMapping) - } -} - -func TestMappingStructWithJSONTags(t *testing.T) { - - mapping := buildMapping() - - x := struct { - NoJSONTag string - Name string `json:"name"` - }{ - Name: "marty", - } - - doc := document.NewDocument("1") - err := mapping.mapDocument(doc, x) - if err != nil { - t.Fatal(err) - } - foundJSONName := false - foundNoJSONName := false - count := 0 - for _, f := range doc.Fields { - if f.Name() == "name" { - foundJSONName = true - } - if f.Name() == "NoJSONTag" { - foundNoJSONName = true - } - count++ - } - if !foundJSONName { - t.Errorf("expected to find field named 'name'") - } - if !foundNoJSONName { - t.Errorf("expected to find field named 'NoJSONTag'") - } - if count != 2 { - t.Errorf("expected to find 2 find, found %d", count) - } -} - -func TestMappingStructWithJSONTagsOneDisabled(t *testing.T) { - - mapping := buildMapping() - - x := struct { - Name string `json:"name"` - Title string `json:"-"` - NoJSONTag string - }{ - Name: "marty", - } - - doc := document.NewDocument("1") - err := mapping.mapDocument(doc, x) - if err != nil { - t.Fatal(err) - } - foundJSONName := false - foundNoJSONName := false - count := 0 - for _, f := range doc.Fields { - if f.Name() == "name" { - foundJSONName = true - } - if f.Name() == "NoJSONTag" { - foundNoJSONName = true - } - count++ - } - if !foundJSONName { - t.Errorf("expected to find field named 'name'") - } - if !foundNoJSONName { - t.Errorf("expected to find field named 'NoJSONTag'") - } - if count != 2 { - t.Errorf("expected to find 2 find, found %d", count) - } -} - -func TestMappingStructWithPointerToString(t *testing.T) { - - mapping := buildMapping() - - name := "marty" - - x := struct { - Name *string - }{ - Name: &name, - } - - doc := document.NewDocument("1") - err := mapping.mapDocument(doc, x) - if err != nil { - t.Fatal(err) - } - found := false - count := 0 - for _, f := range doc.Fields { - if f.Name() == "Name" { - found = true - } - count++ - } - if !found { - t.Errorf("expected to find field named 'Name'") - } - if count != 1 { - t.Errorf("expected to find 1 find, found %d", count) - } -} - -func TestMappingJSONWithNull(t *testing.T) { - - mapping := NewIndexMapping() - - jsonbytes := []byte(`{"name":"marty", "age": null}`) - var jsondoc interface{} - err := json.Unmarshal(jsonbytes, &jsondoc) - if err != nil { - t.Fatal(err) - } - - doc := document.NewDocument("1") - err = mapping.mapDocument(doc, jsondoc) - if err != nil { - t.Fatal(err) - } - found := false - count := 0 - for _, f := range doc.Fields { - if f.Name() == "name" { - found = true - } - count++ - } - if !found { - t.Errorf("expected to find field named 'name'") - } - if count != 1 { - t.Errorf("expected to find 1 find, found %d", count) - } -} - -func TestMappingForPath(t *testing.T) { - - enFieldMapping := NewTextFieldMapping() - enFieldMapping.Analyzer = "en" - - docMappingA := NewDocumentMapping() - docMappingA.AddFieldMappingsAt("name", enFieldMapping) - - customMapping := NewTextFieldMapping() - customMapping.Analyzer = "xyz" - customMapping.Name = "nameCustom" - - docMappingA.AddFieldMappingsAt("author", enFieldMapping, customMapping) - - mapping := NewIndexMapping() - mapping.AddDocumentMapping("a", docMappingA) - - analyzerName := mapping.analyzerNameForPath("name") - if analyzerName != enFieldMapping.Analyzer { - t.Errorf("expected '%s' got '%s'", enFieldMapping.Analyzer, analyzerName) - } - - analyzerName = mapping.analyzerNameForPath("nameCustom") - if analyzerName != customMapping.Analyzer { - t.Errorf("expected '%s' got '%s'", customMapping.Analyzer, analyzerName) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/numeric/bin.go b/vendor/github.com/blevesearch/bleve/numeric/bin.go new file mode 100644 index 00000000..cd71392d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/numeric/bin.go @@ -0,0 +1,43 @@ +package numeric + +var interleaveMagic = []uint64{ + 0x5555555555555555, + 0x3333333333333333, + 0x0F0F0F0F0F0F0F0F, + 0x00FF00FF00FF00FF, + 0x0000FFFF0000FFFF, + 0x00000000FFFFFFFF, + 0xAAAAAAAAAAAAAAAA, +} + +var interleaveShift = []uint{1, 2, 4, 8, 16} + +// Interleave the first 32 bits of each uint64 +// apdated from org.apache.lucene.util.BitUtil +// whcih was adapted from: +// http://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN +func Interleave(v1, v2 uint64) uint64 { + v1 = (v1 | (v1 << interleaveShift[4])) & interleaveMagic[4] + v1 = (v1 | (v1 << interleaveShift[3])) & interleaveMagic[3] + v1 = (v1 | (v1 << interleaveShift[2])) & interleaveMagic[2] + v1 = (v1 | (v1 << interleaveShift[1])) & interleaveMagic[1] + v1 = (v1 | (v1 << interleaveShift[0])) & interleaveMagic[0] + v2 = (v2 | (v2 << interleaveShift[4])) & interleaveMagic[4] + v2 = (v2 | (v2 << interleaveShift[3])) & interleaveMagic[3] + v2 = (v2 | (v2 << interleaveShift[2])) & interleaveMagic[2] + v2 = (v2 | (v2 << interleaveShift[1])) & interleaveMagic[1] + v2 = (v2 | (v2 << interleaveShift[0])) & interleaveMagic[0] + return (v2 << 1) | v1 +} + +// Deinterleave the 32-bit value starting at position 0 +// to get the other 32-bit value, shift it by 1 first +func Deinterleave(b uint64) uint64 { + b &= interleaveMagic[0] + b = (b ^ (b >> interleaveShift[0])) & interleaveMagic[1] + b = (b ^ (b >> interleaveShift[1])) & interleaveMagic[2] + b = (b ^ (b >> interleaveShift[2])) & interleaveMagic[3] + b = (b ^ (b >> interleaveShift[3])) & interleaveMagic[4] + b = (b ^ (b >> interleaveShift[4])) & interleaveMagic[5] + return b +} diff --git a/vendor/github.com/blevesearch/bleve/numeric/float.go b/vendor/github.com/blevesearch/bleve/numeric/float.go new file mode 100644 index 00000000..2bb14d7e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/numeric/float.go @@ -0,0 +1,34 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numeric + +import ( + "math" +) + +func Float64ToInt64(f float64) int64 { + fasint := int64(math.Float64bits(f)) + if fasint < 0 { + fasint = fasint ^ 0x7fffffffffffffff + } + return fasint +} + +func Int64ToFloat64(i int64) float64 { + if i < 0 { + i ^= 0x7fffffffffffffff + } + return math.Float64frombits(uint64(i)) +} diff --git a/vendor/github.com/blevesearch/bleve/numeric_util/prefix_coded.go b/vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go similarity index 61% rename from vendor/github.com/blevesearch/bleve/numeric_util/prefix_coded.go rename to vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go index 263a1cf4..4200c23b 100644 --- a/vendor/github.com/blevesearch/bleve/numeric_util/prefix_coded.go +++ b/vendor/github.com/blevesearch/bleve/numeric/prefix_coded.go @@ -1,17 +1,20 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package numeric_util +package numeric -import ( - "fmt" -) +import "fmt" const ShiftStartInt64 byte = 0x20 @@ -24,7 +27,7 @@ func NewPrefixCodedInt64(in int64, shift uint) (PrefixCoded, error) { return nil, fmt.Errorf("cannot shift %d, must be between 0 and 63", shift) } - nChars := (((63 - shift) * 37) >> 8) + 1 + nChars := ((63 - shift) / 7) + 1 rv := make(PrefixCoded, nChars+1) rv[0] = ShiftStartInt64 + byte(shift) @@ -72,3 +75,18 @@ func (p PrefixCoded) Int64() (int64, error) { } return int64(uint64((sortableBits << shift)) ^ 0x8000000000000000), nil } + +func ValidPrefixCodedTerm(p string) (bool, int) { + if len(p) > 0 { + if p[0] < ShiftStartInt64 || p[0] > ShiftStartInt64+63 { + return false, 0 + } + shift := p[0] - ShiftStartInt64 + nChars := ((63 - int(shift)) / 7) + 1 + if len(p) != nChars+1 { + return false, 0 + } + return true, int(shift) + } + return false, 0 +} diff --git a/vendor/github.com/blevesearch/bleve/numeric_util/float.go b/vendor/github.com/blevesearch/bleve/numeric_util/float.go deleted file mode 100644 index a33ea1af..00000000 --- a/vendor/github.com/blevesearch/bleve/numeric_util/float.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package numeric_util - -import ( - "math" -) - -func Float64ToInt64(f float64) int64 { - fasint := int64(math.Float64bits(f)) - if fasint < 0 { - fasint = fasint ^ 0x7fffffffffffffff - } - return fasint -} - -func Int64ToFloat64(i int64) float64 { - if i < 0 { - i ^= 0x7fffffffffffffff - } - return math.Float64frombits(uint64(i)) -} diff --git a/vendor/github.com/blevesearch/bleve/numeric_util/float_test.go b/vendor/github.com/blevesearch/bleve/numeric_util/float_test.go deleted file mode 100644 index dc7bf496..00000000 --- a/vendor/github.com/blevesearch/bleve/numeric_util/float_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package numeric_util - -import ( - "testing" -) - -// test that the float/sortable int operations work both ways -// and that the corresponding integers sort the same as -// the original floats would have -func TestSortabledFloat64ToInt64(t *testing.T) { - tests := []struct { - input float64 - }{ - { - input: -4640094584139352638, - }, - { - input: -167.42, - }, - { - input: -1.11, - }, - { - input: 0, - }, - { - input: 3.14, - }, - { - input: 167.42, - }, - } - - var lastInt64 *int64 - for _, test := range tests { - actual := Float64ToInt64(test.input) - if lastInt64 != nil { - // check that this float is greater than the last one - if actual <= *lastInt64 { - t.Errorf("expected greater than prev, this: %d, last %d", actual, *lastInt64) - } - } - lastInt64 = &actual - convertedBack := Int64ToFloat64(actual) - // assert that we got back what we started with - if convertedBack != test.input { - t.Errorf("expected %f, got %f", test.input, convertedBack) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/numeric_util/prefix_coded_test.go b/vendor/github.com/blevesearch/bleve/numeric_util/prefix_coded_test.go deleted file mode 100644 index 59110f45..00000000 --- a/vendor/github.com/blevesearch/bleve/numeric_util/prefix_coded_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package numeric_util - -import ( - "reflect" - "testing" -) - -// these array encoding values have been verified manually -// against the lucene implementation -func TestPrefixCoded(t *testing.T) { - tests := []struct { - input int64 - shift uint - output PrefixCoded - }{ - { - input: 1, - shift: 0, - output: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, - }, - { - input: -1, - shift: 0, - output: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, - }, - { - input: -94582, - shift: 0, - output: PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa}, - }, - { - input: 314729851, - shift: 0, - output: PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b}, - }, - { - input: 314729851, - shift: 4, - output: PrefixCoded{0x24, 0x8, 0x0, 0x0, 0x0, 0x0, 0x9, 0x30, 0x4c, 0x57}, - }, - { - input: 314729851, - shift: 8, - output: PrefixCoded{0x28, 0x40, 0x0, 0x0, 0x0, 0x0, 0x4b, 0x4, 0x65}, - }, - { - input: 314729851, - shift: 16, - output: PrefixCoded{0x30, 0x20, 0x0, 0x0, 0x0, 0x0, 0x25, 0x42}, - }, - { - input: 314729851, - shift: 32, - output: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0}, - }, - { - input: 1234729851, - shift: 32, - output: PrefixCoded{0x40, 0x8, 0x0, 0x0, 0x0, 0x0}, - }, - } - - for _, test := range tests { - actual, err := NewPrefixCodedInt64(test.input, test.shift) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(actual, test.output) { - t.Errorf("expected %#v, got %#v", test.output, actual) - } - checkedShift, err := actual.Shift() - if err != nil { - t.Error(err) - } - if checkedShift != test.shift { - t.Errorf("expected %d, got %d", test.shift, checkedShift) - } - // if the shift was 0, make sure we can go back to the original - if test.shift == 0 { - backToLong, err := actual.Int64() - if err != nil { - t.Error(err) - } - if backToLong != test.input { - t.Errorf("expected %v, got %v", test.input, backToLong) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/query.go b/vendor/github.com/blevesearch/bleve/query.go index b2c97989..1fecfa25 100644 --- a/vendor/github.com/blevesearch/bleve/query.go +++ b/vendor/github.com/blevesearch/bleve/query.go @@ -1,211 +1,218 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve import ( - "encoding/json" + "time" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/query" ) -// A Query represents a description of the type -// and parameters for a query into the index. -type Query interface { - Boost() float64 - SetBoost(b float64) Query - Field() string - SetField(f string) Query - Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) - Validate() error +// NewBoolFieldQuery creates a new Query for boolean fields +func NewBoolFieldQuery(val bool) *query.BoolFieldQuery { + return query.NewBoolFieldQuery(val) } -// ParseQuery deserializes a JSON representation of -// a Query object. -func ParseQuery(input []byte) (Query, error) { - var tmp map[string]interface{} - err := json.Unmarshal(input, &tmp) - if err != nil { - return nil, err - } - _, isMatchQuery := tmp["match"] - _, hasFuzziness := tmp["fuzziness"] - if hasFuzziness && !isMatchQuery { - var rv fuzzyQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, isTermQuery := tmp["term"] - if isTermQuery { - var rv termQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - if isMatchQuery { - var rv matchQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, isMatchPhraseQuery := tmp["match_phrase"] - if isMatchPhraseQuery { - var rv matchPhraseQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, hasMust := tmp["must"] - _, hasShould := tmp["should"] - _, hasMustNot := tmp["must_not"] - if hasMust || hasShould || hasMustNot { - var rv booleanQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, hasTerms := tmp["terms"] - if hasTerms { - var rv phraseQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - for _, tq := range rv.TermQueries { - if tq.Boost() == 0 { - tq.SetBoost(1) - } - } - return &rv, nil - } - _, hasConjuncts := tmp["conjuncts"] - if hasConjuncts { - var rv conjunctionQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, hasDisjuncts := tmp["disjuncts"] - if hasDisjuncts { - var rv disjunctionQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - - _, hasSyntaxQuery := tmp["query"] - if hasSyntaxQuery { - var rv queryStringQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, hasMin := tmp["min"] - _, hasMax := tmp["max"] - if hasMin || hasMax { - var rv numericRangeQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, hasStart := tmp["start"] - _, hasEnd := tmp["end"] - if hasStart || hasEnd { - var rv dateRangeQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, hasPrefix := tmp["prefix"] - if hasPrefix { - var rv prefixQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - if rv.Boost() == 0 { - rv.SetBoost(1) - } - return &rv, nil - } - _, hasRegexp := tmp["regexp"] - if hasRegexp { - var rv regexpQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - _, hasWildcard := tmp["wildcard"] - if hasWildcard { - var rv wildcardQuery - err := json.Unmarshal(input, &rv) - if err != nil { - return nil, err - } - return &rv, nil - } - return nil, ErrorUnknownQueryType +// NewBooleanQuery creates a compound Query composed +// of several other Query objects. +// These other query objects are added using the +// AddMust() AddShould() and AddMustNot() methods. +// Result documents must satisfy ALL of the +// must Queries. +// Result documents must satisfy NONE of the must not +// Queries. +// Result documents that ALSO satisfy any of the should +// Queries will score higher. +func NewBooleanQuery() *query.BooleanQuery { + return query.NewBooleanQuery(nil, nil, nil) +} + +// NewConjunctionQuery creates a new compound Query. +// Result documents must satisfy all of the queries. +func NewConjunctionQuery(conjuncts ...query.Query) *query.ConjunctionQuery { + return query.NewConjunctionQuery(conjuncts) +} + +// NewDateRangeQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// top-level config.QueryDateTimeParser +// Either, but not both endpoints can be nil. +func NewDateRangeQuery(start, end time.Time) *query.DateRangeQuery { + return query.NewDateRangeQuery(start, end) +} + +// NewDateRangeInclusiveQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// top-level config.QueryDateTimeParser +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *query.DateRangeQuery { + return query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive) +} + +// NewDisjunctionQuery creates a new compound Query. +// Result documents satisfy at least one Query. +func NewDisjunctionQuery(disjuncts ...query.Query) *query.DisjunctionQuery { + return query.NewDisjunctionQuery(disjuncts) +} + +// NewDocIDQuery creates a new Query object returning indexed documents among +// the specified set. Combine it with ConjunctionQuery to restrict the scope of +// other queries output. +func NewDocIDQuery(ids []string) *query.DocIDQuery { + return query.NewDocIDQuery(ids) +} + +// NewFuzzyQuery creates a new Query which finds +// documents containing terms within a specific +// fuzziness of the specified term. +// The default fuzziness is 1. +// +// The current implementation uses Levenshtein edit +// distance as the fuzziness metric. +func NewFuzzyQuery(term string) *query.FuzzyQuery { + return query.NewFuzzyQuery(term) +} + +// NewMatchAllQuery creates a Query which will +// match all documents in the index. +func NewMatchAllQuery() *query.MatchAllQuery { + return query.NewMatchAllQuery() +} + +// NewMatchNoneQuery creates a Query which will not +// match any documents in the index. +func NewMatchNoneQuery() *query.MatchNoneQuery { + return query.NewMatchNoneQuery() +} + +// NewMatchPhraseQuery creates a new Query object +// for matching phrases in the index. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to build a search phrase. Result documents +// must match this phrase. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMatchPhraseQuery(matchPhrase string) *query.MatchPhraseQuery { + return query.NewMatchPhraseQuery(matchPhrase) +} + +// NewMatchQuery creates a Query for matching text. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to perform term searches. Result documents +// must satisfy at least one of these term searches. +func NewMatchQuery(match string) *query.MatchQuery { + return query.NewMatchQuery(match) +} + +// NewNumericRangeQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewNumericRangeQuery(min, max *float64) *query.NumericRangeQuery { + return query.NewNumericRangeQuery(min, max) +} + +// NewNumericRangeInclusiveQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *query.NumericRangeQuery { + return query.NewNumericRangeInclusiveQuery(min, max, minInclusive, maxInclusive) +} + +// NewTermRangeQuery creates a new Query for ranges +// of text terms. +// Either, but not both endpoints can be "". +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewTermRangeQuery(min, max string) *query.TermRangeQuery { + return query.NewTermRangeQuery(min, max) +} + +// NewTermRangeInclusiveQuery creates a new Query for ranges +// of text terms. +// Either, but not both endpoints can be "". +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *query.TermRangeQuery { + return query.NewTermRangeInclusiveQuery(min, max, minInclusive, maxInclusive) +} + +// NewPhraseQuery creates a new Query for finding +// exact term phrases in the index. +// The provided terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewPhraseQuery(terms []string, field string) *query.PhraseQuery { + return query.NewPhraseQuery(terms, field) +} + +// NewPrefixQuery creates a new Query which finds +// documents containing terms that start with the +// specified prefix. +func NewPrefixQuery(prefix string) *query.PrefixQuery { + return query.NewPrefixQuery(prefix) +} + +// NewRegexpQuery creates a new Query which finds +// documents containing terms that match the +// specified regular expression. +func NewRegexpQuery(regexp string) *query.RegexpQuery { + return query.NewRegexpQuery(regexp) +} + +// NewQueryStringQuery creates a new Query used for +// finding documents that satisfy a query string. The +// query string is a small query language for humans. +func NewQueryStringQuery(q string) *query.QueryStringQuery { + return query.NewQueryStringQuery(q) +} + +// NewTermQuery creates a new Query for finding an +// exact term match in the index. +func NewTermQuery(term string) *query.TermQuery { + return query.NewTermQuery(term) +} + +// NewWildcardQuery creates a new Query which finds +// documents containing terms that match the +// specified wildcard. In the wildcard pattern '*' +// will match any sequence of 0 or more characters, +// and '?' will match any single character. +func NewWildcardQuery(wildcard string) *query.WildcardQuery { + return query.NewWildcardQuery(wildcard) +} + +// NewGeoBoundingBoxQuery creates a new Query for performing geo bounding +// box searches. The arguments describe the position of the box and documents +// which have an indexed geo point inside the box will be returned. +func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *query.GeoBoundingBoxQuery { + return query.NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat) +} + +// NewGeoDistanceQuery creates a new Query for performing geo bounding +// box searches. The arguments describe a position and a distance. Documents +// which have an indexed geo point which is less than or equal to the provided +// distance from the given position will be returned. +func NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery { + return query.NewGeoDistanceQuery(lon, lat, distance) } diff --git a/vendor/github.com/blevesearch/bleve/query_boolean.go b/vendor/github.com/blevesearch/bleve/query_boolean.go deleted file mode 100644 index bf540148..00000000 --- a/vendor/github.com/blevesearch/bleve/query_boolean.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type booleanQuery struct { - Must Query `json:"must,omitempty"` - Should Query `json:"should,omitempty"` - MustNot Query `json:"must_not,omitempty"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewBooleanQuery creates a compound Query composed -// of several other Query objects. -// Result documents must satisfy ALL of the -// must Queries. -// Result documents must satisfy NONE of the must not -// Queries. -// If there are any should queries, result documents -// must satisfy at least one of them. -func NewBooleanQuery(must []Query, should []Query, mustNot []Query) *booleanQuery { - min := 0.0 - if len(should) > 0 { - min = 1.0 - } - return NewBooleanQueryMinShould(must, should, mustNot, min) -} - -// NewBooleanQueryMinShould is the same as -// NewBooleanQuery, only it offers control of the -// minimum number of should queries that must be -// satisfied. -func NewBooleanQueryMinShould(must []Query, should []Query, mustNot []Query, minShould float64) *booleanQuery { - - rv := booleanQuery{ - BoostVal: 1.0, - } - if len(must) > 0 { - rv.Must = NewConjunctionQuery(must) - } - if len(should) > 0 { - rv.Should = NewDisjunctionQueryMin(should, minShould) - } - if len(mustNot) > 0 { - rv.MustNot = NewDisjunctionQuery(mustNot) - } - - return &rv -} - -func (q *booleanQuery) AddMust(m Query) { - if q.Must == nil { - q.Must = NewConjunctionQuery([]Query{}) - } - q.Must.(*conjunctionQuery).AddQuery(m) -} - -func (q *booleanQuery) AddShould(m Query) { - if q.Should == nil { - q.Should = NewDisjunctionQuery([]Query{}) - } - q.Should.(*disjunctionQuery).AddQuery(m) - q.Should.(*disjunctionQuery).SetMin(1) -} - -func (q *booleanQuery) AddMustNot(m Query) { - if q.MustNot == nil { - q.MustNot = NewDisjunctionQuery([]Query{}) - } - q.MustNot.(*disjunctionQuery).AddQuery(m) -} - -func (q *booleanQuery) Boost() float64 { - return q.BoostVal -} - -func (q *booleanQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *booleanQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - var err error - - var mustSearcher search.Searcher - if q.Must != nil { - mustSearcher, err = q.Must.Searcher(i, m, explain) - if err != nil { - return nil, err - } - } - - var shouldSearcher search.Searcher - if q.Should != nil { - shouldSearcher, err = q.Should.Searcher(i, m, explain) - if err != nil { - return nil, err - } - } - - var mustNotSearcher search.Searcher - if q.MustNot != nil { - mustNotSearcher, err = q.MustNot.Searcher(i, m, explain) - if err != nil { - return nil, err - } - } - - return searchers.NewBooleanSearcher(i, mustSearcher, shouldSearcher, mustNotSearcher, explain) -} - -func (q *booleanQuery) Validate() error { - if q.Must != nil { - err := q.Must.Validate() - if err != nil { - return err - } - } - if q.Should != nil { - err := q.Should.Validate() - if err != nil { - return err - } - } - if q.MustNot != nil { - err := q.MustNot.Validate() - if err != nil { - return err - } - } - if q.Must == nil && q.Should == nil { - return ErrorBooleanQueryNeedsMustOrShould - } - return nil -} - -func (q *booleanQuery) UnmarshalJSON(data []byte) error { - tmp := struct { - Must json.RawMessage `json:"must,omitempty"` - Should json.RawMessage `json:"should,omitempty"` - MustNot json.RawMessage `json:"must_not,omitempty"` - BoostVal float64 `json:"boost,omitempty"` - }{} - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - - if tmp.Must != nil { - q.Must, err = ParseQuery(tmp.Must) - if err != nil { - return err - } - _, isConjunctionQuery := q.Must.(*conjunctionQuery) - if !isConjunctionQuery { - return fmt.Errorf("must clause must be conjunction") - } - } - - if tmp.Should != nil { - q.Should, err = ParseQuery(tmp.Should) - if err != nil { - return err - } - _, isDisjunctionQuery := q.Should.(*disjunctionQuery) - if !isDisjunctionQuery { - return fmt.Errorf("should clause must be disjunction") - } - } - - if tmp.MustNot != nil { - q.MustNot, err = ParseQuery(tmp.MustNot) - if err != nil { - return err - } - _, isDisjunctionQuery := q.MustNot.(*disjunctionQuery) - if !isDisjunctionQuery { - return fmt.Errorf("must not clause must be disjunction") - } - } - - q.BoostVal = tmp.BoostVal - if q.BoostVal == 0 { - q.BoostVal = 1 - } - return nil -} - -func (q *booleanQuery) Field() string { - return "" -} - -func (q *booleanQuery) SetField(f string) Query { - return q -} diff --git a/vendor/github.com/blevesearch/bleve/query_conjunction.go b/vendor/github.com/blevesearch/bleve/query_conjunction.go deleted file mode 100644 index 75018910..00000000 --- a/vendor/github.com/blevesearch/bleve/query_conjunction.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "encoding/json" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type conjunctionQuery struct { - Conjuncts []Query `json:"conjuncts"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewConjunctionQuery creates a new compound Query. -// Result documents must satisfy all of the queries. -func NewConjunctionQuery(conjuncts []Query) *conjunctionQuery { - return &conjunctionQuery{ - Conjuncts: conjuncts, - BoostVal: 1.0, - } -} - -func (q *conjunctionQuery) Boost() float64 { - return q.BoostVal -} - -func (q *conjunctionQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *conjunctionQuery) AddQuery(aq Query) *conjunctionQuery { - q.Conjuncts = append(q.Conjuncts, aq) - return q -} - -func (q *conjunctionQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - ss := make([]search.Searcher, len(q.Conjuncts)) - for in, conjunct := range q.Conjuncts { - var err error - ss[in], err = conjunct.Searcher(i, m, explain) - if err != nil { - return nil, err - } - } - return searchers.NewConjunctionSearcher(i, ss, explain) -} - -func (q *conjunctionQuery) Validate() error { - return nil -} - -func (q *conjunctionQuery) UnmarshalJSON(data []byte) error { - tmp := struct { - Conjuncts []json.RawMessage `json:"conjuncts"` - BoostVal float64 `json:"boost,omitempty"` - }{} - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - q.Conjuncts = make([]Query, len(tmp.Conjuncts)) - for i, term := range tmp.Conjuncts { - query, err := ParseQuery(term) - if err != nil { - return err - } - q.Conjuncts[i] = query - } - q.BoostVal = tmp.BoostVal - if q.BoostVal == 0 { - q.BoostVal = 1 - } - return nil -} - -func (q *conjunctionQuery) Field() string { - return "" -} - -func (q *conjunctionQuery) SetField(f string) Query { - return q -} diff --git a/vendor/github.com/blevesearch/bleve/query_date_range.go b/vendor/github.com/blevesearch/bleve/query_date_range.go deleted file mode 100644 index 8643d83d..00000000 --- a/vendor/github.com/blevesearch/bleve/query_date_range.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "fmt" - "math" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric_util" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type dateRangeQuery struct { - Start *string `json:"start,omitempty"` - End *string `json:"end,omitempty"` - InclusiveStart *bool `json:"inclusive_start,omitempty"` - InclusiveEnd *bool `json:"inclusive_end,omitempty"` - FieldVal string `json:"field,omitempty"` - BoostVal float64 `json:"boost,omitempty"` - DateTimeParser *string `json:"datetime_parser,omitempty"` -} - -// NewDateRangeQuery creates a new Query for ranges -// of date values. -// A DateTimeParser is chosen based on the field. -// Either, but not both endpoints can be nil. -func NewDateRangeQuery(start, end *string) *dateRangeQuery { - return NewDateRangeInclusiveQuery(start, end, nil, nil) -} - -// NewDateRangeInclusiveQuery creates a new Query for ranges -// of date values. -// A DateTimeParser is chosen based on the field. -// Either, but not both endpoints can be nil. -// startInclusive and endInclusive control inclusion of the endpoints. -func NewDateRangeInclusiveQuery(start, end *string, startInclusive, endInclusive *bool) *dateRangeQuery { - return &dateRangeQuery{ - Start: start, - End: end, - InclusiveStart: startInclusive, - InclusiveEnd: endInclusive, - BoostVal: 1.0, - } -} - -func (q *dateRangeQuery) Boost() float64 { - return q.BoostVal -} - -func (q *dateRangeQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *dateRangeQuery) Field() string { - return q.FieldVal -} - -func (q *dateRangeQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *dateRangeQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - - dateTimeParserName := "" - if q.DateTimeParser != nil { - dateTimeParserName = *q.DateTimeParser - } else { - dateTimeParserName = m.datetimeParserNameForPath(q.FieldVal) - } - dateTimeParser := m.dateTimeParserNamed(dateTimeParserName) - if dateTimeParser == nil { - return nil, fmt.Errorf("no datetime parser named '%s' registered", *q.DateTimeParser) - } - - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - - // now parse the endpoints - min := math.Inf(-1) - max := math.Inf(1) - if q.Start != nil && *q.Start != "" { - startTime, err := dateTimeParser.ParseDateTime(*q.Start) - if err != nil { - return nil, err - } - min = numeric_util.Int64ToFloat64(startTime.UnixNano()) - } - if q.End != nil && *q.End != "" { - endTime, err := dateTimeParser.ParseDateTime(*q.End) - if err != nil { - return nil, err - } - max = numeric_util.Int64ToFloat64(endTime.UnixNano()) - } - - return searchers.NewNumericRangeSearcher(i, &min, &max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal, explain) -} - -func (q *dateRangeQuery) Validate() error { - if q.Start == nil && q.Start == q.End { - return fmt.Errorf("must specify start or end") - } - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/query_disjunction.go b/vendor/github.com/blevesearch/bleve/query_disjunction.go deleted file mode 100644 index 7f9faefa..00000000 --- a/vendor/github.com/blevesearch/bleve/query_disjunction.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "encoding/json" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type disjunctionQuery struct { - Disjuncts []Query `json:"disjuncts"` - BoostVal float64 `json:"boost,omitempty"` - MinVal float64 `json:"min"` -} - -// NewDisjunctionQuery creates a new compound Query. -// Result documents satisfy at least one Query. -func NewDisjunctionQuery(disjuncts []Query) *disjunctionQuery { - return &disjunctionQuery{ - Disjuncts: disjuncts, - BoostVal: 1.0, - } -} - -// NewDisjunctionQueryMin creates a new compound Query. -// Result documents satisfy at least min Queries. -func NewDisjunctionQueryMin(disjuncts []Query, min float64) *disjunctionQuery { - return &disjunctionQuery{ - Disjuncts: disjuncts, - BoostVal: 1.0, - MinVal: min, - } -} - -func (q *disjunctionQuery) Boost() float64 { - return q.BoostVal -} - -func (q *disjunctionQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *disjunctionQuery) AddQuery(aq Query) Query { - q.Disjuncts = append(q.Disjuncts, aq) - return q -} - -func (q *disjunctionQuery) Min() float64 { - return q.MinVal -} - -func (q *disjunctionQuery) SetMin(m float64) Query { - q.MinVal = m - return q -} - -func (q *disjunctionQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - ss := make([]search.Searcher, len(q.Disjuncts)) - for in, disjunct := range q.Disjuncts { - var err error - ss[in], err = disjunct.Searcher(i, m, explain) - if err != nil { - return nil, err - } - } - return searchers.NewDisjunctionSearcher(i, ss, q.MinVal, explain) -} - -func (q *disjunctionQuery) Validate() error { - if int(q.MinVal) > len(q.Disjuncts) { - return ErrorDisjunctionFewerThanMinClauses - } - return nil -} - -func (q *disjunctionQuery) UnmarshalJSON(data []byte) error { - tmp := struct { - Disjuncts []json.RawMessage `json:"disjuncts"` - BoostVal float64 `json:"boost,omitempty"` - MinVal float64 `json:"min"` - }{} - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - q.Disjuncts = make([]Query, len(tmp.Disjuncts)) - for i, term := range tmp.Disjuncts { - query, err := ParseQuery(term) - if err != nil { - return err - } - q.Disjuncts[i] = query - } - q.BoostVal = tmp.BoostVal - if q.BoostVal == 0 { - q.BoostVal = 1 - } - q.MinVal = tmp.MinVal - return nil -} - -func (q *disjunctionQuery) Field() string { - return "" -} - -func (q *disjunctionQuery) SetField(f string) Query { - return q -} diff --git a/vendor/github.com/blevesearch/bleve/query_fuzzy.go b/vendor/github.com/blevesearch/bleve/query_fuzzy.go deleted file mode 100644 index 72624e31..00000000 --- a/vendor/github.com/blevesearch/bleve/query_fuzzy.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type fuzzyQuery struct { - Term string `json:"term"` - PrefixVal int `json:"prefix_length"` - FuzzinessVal int `json:"fuzziness"` - FieldVal string `json:"field,omitempty"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewFuzzyQuery creates a new Query which finds -// documents containing terms within a specific -// fuzziness of the specified term. -// The default fuzziness is 2. -// -// The current implementation uses Leveshtein edit -// distance as the fuzziness metric. -func NewFuzzyQuery(term string) *fuzzyQuery { - return &fuzzyQuery{ - Term: term, - PrefixVal: 0, - FuzzinessVal: 2, - BoostVal: 1.0, - } -} - -func (q *fuzzyQuery) Boost() float64 { - return q.BoostVal -} - -func (q *fuzzyQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *fuzzyQuery) Field() string { - return q.FieldVal -} - -func (q *fuzzyQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *fuzzyQuery) Fuzziness() int { - return q.FuzzinessVal -} - -func (q *fuzzyQuery) SetFuzziness(f int) Query { - q.FuzzinessVal = f - return q -} - -func (q *fuzzyQuery) Prefix() int { - return q.PrefixVal -} - -func (q *fuzzyQuery) SetPrefix(p int) Query { - q.PrefixVal = p - return q -} - -func (q *fuzzyQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - return searchers.NewFuzzySearcher(i, q.Term, q.PrefixVal, q.FuzzinessVal, field, q.BoostVal, explain) -} - -func (q *fuzzyQuery) Validate() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/query_match.go b/vendor/github.com/blevesearch/bleve/query_match.go deleted file mode 100644 index 582dbd69..00000000 --- a/vendor/github.com/blevesearch/bleve/query_match.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type matchQuery struct { - Match string `json:"match"` - FieldVal string `json:"field,omitempty"` - Analyzer string `json:"analyzer,omitempty"` - BoostVal float64 `json:"boost,omitempty"` - PrefixVal int `json:"prefix_length"` - FuzzinessVal int `json:"fuzziness"` -} - -// NewMatchQuery creates a Query for matching text. -// An Analyzer is chosen based on the field. -// Input text is analyzed using this analyzer. -// Token terms resulting from this analysis are -// used to perform term searches. Result documents -// must satisfy at least one of these term searches. -func NewMatchQuery(match string) *matchQuery { - return &matchQuery{ - Match: match, - BoostVal: 1.0, - } -} - -func (q *matchQuery) Boost() float64 { - return q.BoostVal -} - -func (q *matchQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *matchQuery) Field() string { - return q.FieldVal -} - -func (q *matchQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *matchQuery) Fuzziness() int { - return q.FuzzinessVal -} - -func (q *matchQuery) SetFuzziness(f int) Query { - q.FuzzinessVal = f - return q -} - -func (q *matchQuery) Prefix() int { - return q.PrefixVal -} - -func (q *matchQuery) SetPrefix(p int) Query { - q.PrefixVal = p - return q -} - -func (q *matchQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - - analyzerName := "" - if q.Analyzer != "" { - analyzerName = q.Analyzer - } else { - analyzerName = m.analyzerNameForPath(field) - } - analyzer := m.analyzerNamed(analyzerName) - - if analyzer == nil { - return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) - } - - tokens := analyzer.Analyze([]byte(q.Match)) - if len(tokens) > 0 { - - tqs := make([]Query, len(tokens)) - if q.FuzzinessVal != 0 { - for i, token := range tokens { - query := NewFuzzyQuery(string(token.Term)) - query.SetFuzziness(q.FuzzinessVal) - query.SetPrefix(q.PrefixVal) - query.SetField(field) - query.SetBoost(q.BoostVal) - tqs[i] = query - } - } else { - for i, token := range tokens { - tqs[i] = NewTermQuery(string(token.Term)). - SetField(field). - SetBoost(q.BoostVal) - } - } - - shouldQuery := NewDisjunctionQueryMin(tqs, 1). - SetBoost(q.BoostVal) - - return shouldQuery.Searcher(i, m, explain) - } - noneQuery := NewMatchNoneQuery() - return noneQuery.Searcher(i, m, explain) -} - -func (q *matchQuery) Validate() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/query_match_all.go b/vendor/github.com/blevesearch/bleve/query_match_all.go deleted file mode 100644 index c7dbcd3f..00000000 --- a/vendor/github.com/blevesearch/bleve/query_match_all.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type matchAllQuery struct { - BoostVal float64 `json:"boost,omitempty"` -} - -// NewMatchAllQuery creates a Query which will -// match all documents in the index. -func NewMatchAllQuery() *matchAllQuery { - return &matchAllQuery{ - BoostVal: 1.0, - } -} - -func (q *matchAllQuery) Boost() float64 { - return q.BoostVal -} - -func (q *matchAllQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *matchAllQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - return searchers.NewMatchAllSearcher(i, q.BoostVal, explain) -} - -func (q *matchAllQuery) Validate() error { - return nil -} - -func (q *matchAllQuery) Field() string { - return "" -} - -func (q *matchAllQuery) SetField(f string) Query { - return q -} diff --git a/vendor/github.com/blevesearch/bleve/query_match_none.go b/vendor/github.com/blevesearch/bleve/query_match_none.go deleted file mode 100644 index 9b4ea8ae..00000000 --- a/vendor/github.com/blevesearch/bleve/query_match_none.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type matchNoneQuery struct { - BoostVal float64 `json:"boost,omitempty"` -} - -// NewMatchNoneQuery creates a Query which will not -// match any documents in the index. -func NewMatchNoneQuery() *matchNoneQuery { - return &matchNoneQuery{ - BoostVal: 1.0, - } -} - -func (q *matchNoneQuery) Boost() float64 { - return q.BoostVal -} - -func (q *matchNoneQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *matchNoneQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - return searchers.NewMatchNoneSearcher(i) -} - -func (q *matchNoneQuery) Validate() error { - return nil -} - -func (q *matchNoneQuery) Field() string { - return "" -} - -func (q *matchNoneQuery) SetField(f string) Query { - return q -} diff --git a/vendor/github.com/blevesearch/bleve/query_match_phrase.go b/vendor/github.com/blevesearch/bleve/query_match_phrase.go deleted file mode 100644 index 5eed8a2a..00000000 --- a/vendor/github.com/blevesearch/bleve/query_match_phrase.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type matchPhraseQuery struct { - MatchPhrase string `json:"match_phrase"` - FieldVal string `json:"field,omitempty"` - Analyzer string `json:"analyzer,omitempty"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewMatchPhraseQuery creates a new Query object -// for matching phrases in the index. -// An Analyzer is chosen based on the field. -// Input text is analyzed using this analyzer. -// Token terms resulting from this analysis are -// used to build a search phrase. Result documents -// must match this phrase. -func NewMatchPhraseQuery(matchPhrase string) *matchPhraseQuery { - return &matchPhraseQuery{ - MatchPhrase: matchPhrase, - BoostVal: 1.0, - } -} - -func (q *matchPhraseQuery) Boost() float64 { - return q.BoostVal -} - -func (q *matchPhraseQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *matchPhraseQuery) Field() string { - return q.FieldVal -} - -func (q *matchPhraseQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *matchPhraseQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - - analyzerName := "" - if q.Analyzer != "" { - analyzerName = q.Analyzer - } else { - analyzerName = m.analyzerNameForPath(field) - } - analyzer := m.analyzerNamed(analyzerName) - if analyzer == nil { - return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) - } - - tokens := analyzer.Analyze([]byte(q.MatchPhrase)) - if len(tokens) > 0 { - phrase := tokenStreamToPhrase(tokens) - phraseQuery := NewPhraseQuery(phrase, field).SetBoost(q.BoostVal) - return phraseQuery.Searcher(i, m, explain) - } - noneQuery := NewMatchNoneQuery() - return noneQuery.Searcher(i, m, explain) -} - -func tokenStreamToPhrase(tokens analysis.TokenStream) []string { - firstPosition := int(^uint(0) >> 1) - lastPosition := 0 - for _, token := range tokens { - if token.Position < firstPosition { - firstPosition = token.Position - } - if token.Position > lastPosition { - lastPosition = token.Position - } - } - phraseLen := lastPosition - firstPosition + 1 - if phraseLen > 0 { - rv := make([]string, phraseLen) - for i := 0; i < phraseLen; i++ { - rv[i] = "" - } - for _, token := range tokens { - pos := token.Position - firstPosition - rv[pos] = string(token.Term) - } - return rv - } - return nil -} - -func (q *matchPhraseQuery) Validate() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/query_numeric_range.go b/vendor/github.com/blevesearch/bleve/query_numeric_range.go deleted file mode 100644 index 317f1f4b..00000000 --- a/vendor/github.com/blevesearch/bleve/query_numeric_range.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type numericRangeQuery struct { - Min *float64 `json:"min,omitempty"` - Max *float64 `json:"max,omitempty"` - InclusiveMin *bool `json:"inclusive_min,omitempty"` - InclusiveMax *bool `json:"inclusive_max,omitempty"` - FieldVal string `json:"field,omitempty"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewNumericRangeQuery creates a new Query for ranges -// of date values. -// Either, but not both endpoints can be nil. -// The minimum value is inclusive. -// The maximum value is exclusive. -func NewNumericRangeQuery(min, max *float64) *numericRangeQuery { - return NewNumericRangeInclusiveQuery(min, max, nil, nil) -} - -// NewNumericRangeInclusiveQuery creates a new Query for ranges -// of date values. -// Either, but not both endpoints can be nil. -// Control endpoint inclusion with inclusiveMin, inclusiveMax. -func NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *numericRangeQuery { - return &numericRangeQuery{ - Min: min, - Max: max, - InclusiveMin: minInclusive, - InclusiveMax: maxInclusive, - BoostVal: 1.0, - } -} - -func (q *numericRangeQuery) Boost() float64 { - return q.BoostVal -} - -func (q *numericRangeQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *numericRangeQuery) Field() string { - return q.FieldVal -} - -func (q *numericRangeQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *numericRangeQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - return searchers.NewNumericRangeSearcher(i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal, explain) -} - -func (q *numericRangeQuery) Validate() error { - if q.Min == nil && q.Min == q.Max { - return ErrorNumericQueryNoBounds - } - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/query_phrase.go b/vendor/github.com/blevesearch/bleve/query_phrase.go deleted file mode 100644 index d0318146..00000000 --- a/vendor/github.com/blevesearch/bleve/query_phrase.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "encoding/json" - "fmt" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type phraseQuery struct { - TermQueries []Query `json:"terms"` - BoostVal float64 `json:"boost,omitempty"` - terms []string -} - -// NewPhraseQuery creates a new Query for finding -// exact term phrases in the index. -// The provided terms must exist in the correct -// order, at the correct index offsets, in the -// specified field. -func NewPhraseQuery(terms []string, field string) *phraseQuery { - termQueries := make([]Query, 0) - for _, term := range terms { - if term != "" { - termQueries = append(termQueries, NewTermQuery(term).SetField(field)) - } - } - return &phraseQuery{ - TermQueries: termQueries, - BoostVal: 1.0, - terms: terms, - } -} - -func (q *phraseQuery) Boost() float64 { - return q.BoostVal -} - -func (q *phraseQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *phraseQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - - conjunctionQuery := NewConjunctionQuery(q.TermQueries) - conjunctionSearcher, err := conjunctionQuery.Searcher(i, m, explain) - if err != nil { - return nil, err - } - return searchers.NewPhraseSearcher(i, conjunctionSearcher.(*searchers.ConjunctionSearcher), q.terms) -} - -func (q *phraseQuery) Validate() error { - if len(q.TermQueries) < 1 { - return ErrorPhraseQueryNoTerms - } - return nil -} - -func (q *phraseQuery) UnmarshalJSON(data []byte) error { - tmp := struct { - Terms []json.RawMessage `json:"terms"` - BoostVal float64 `json:"boost,omitempty"` - }{} - err := json.Unmarshal(data, &tmp) - if err != nil { - return err - } - q.TermQueries = make([]Query, len(tmp.Terms)) - q.terms = make([]string, 0) - for i, term := range tmp.Terms { - query, err := ParseQuery(term) - if err != nil { - return err - } - q.TermQueries[i] = query - tq, isTermQuery := query.(*termQuery) - if !isTermQuery { - return fmt.Errorf("phrase query can only contain term queries") - } - q.terms = append(q.terms, tq.Term) - } - q.BoostVal = tmp.BoostVal - if q.BoostVal == 0 { - q.BoostVal = 1 - } - return nil -} - -func (q *phraseQuery) Field() string { - return "" -} - -func (q *phraseQuery) SetField(f string) Query { - return q -} diff --git a/vendor/github.com/blevesearch/bleve/query_prefix.go b/vendor/github.com/blevesearch/bleve/query_prefix.go deleted file mode 100644 index 8d032e54..00000000 --- a/vendor/github.com/blevesearch/bleve/query_prefix.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type prefixQuery struct { - Prefix string `json:"prefix"` - FieldVal string `json:"field,omitempty"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewPrefixQuery creates a new Query which finds -// documents containing terms that start with the -// specified prefix. -func NewPrefixQuery(prefix string) *prefixQuery { - return &prefixQuery{ - Prefix: prefix, - BoostVal: 1.0, - } -} - -func (q *prefixQuery) Boost() float64 { - return q.BoostVal -} - -func (q *prefixQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *prefixQuery) Field() string { - return q.FieldVal -} - -func (q *prefixQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *prefixQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - return searchers.NewTermPrefixSearcher(i, q.Prefix, field, q.BoostVal, explain) -} - -func (q *prefixQuery) Validate() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/query_regexp.go b/vendor/github.com/blevesearch/bleve/query_regexp.go deleted file mode 100644 index ee53b00e..00000000 --- a/vendor/github.com/blevesearch/bleve/query_regexp.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "regexp" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type regexpQuery struct { - Regexp string `json:"regexp"` - FieldVal string `json:"field,omitempty"` - BoostVal float64 `json:"boost,omitempty"` - compiled *regexp.Regexp -} - -// NewRegexpQuery creates a new Query which finds -// documents containing terms that match the -// specified regular expression. -func NewRegexpQuery(regexp string) *regexpQuery { - return ®expQuery{ - Regexp: regexp, - BoostVal: 1.0, - } -} - -func (q *regexpQuery) Boost() float64 { - return q.BoostVal -} - -func (q *regexpQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *regexpQuery) Field() string { - return q.FieldVal -} - -func (q *regexpQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *regexpQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - if q.compiled == nil { - var err error - q.compiled, err = regexp.Compile(q.Regexp) - if err != nil { - return nil, err - } - } - - return searchers.NewRegexpSearcher(i, q.compiled, field, q.BoostVal, explain) -} - -func (q *regexpQuery) Validate() error { - var err error - q.compiled, err = regexp.Compile(q.Regexp) - return err -} diff --git a/vendor/github.com/blevesearch/bleve/query_string.go b/vendor/github.com/blevesearch/bleve/query_string.go deleted file mode 100644 index 15e077e1..00000000 --- a/vendor/github.com/blevesearch/bleve/query_string.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type queryStringQuery struct { - Query string `json:"query"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewQueryStringQuery creates a new Query used for -// finding documents that satisfy a query string. The -// query string is a small query language for humans. -func NewQueryStringQuery(query string) *queryStringQuery { - return &queryStringQuery{ - Query: query, - BoostVal: 1.0, - } -} - -func (q *queryStringQuery) Boost() float64 { - return q.BoostVal -} - -func (q *queryStringQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *queryStringQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - newQuery, err := parseQuerySyntax(q.Query, m) - if err != nil { - return nil, err - } - return newQuery.Searcher(i, m, explain) -} - -func (q *queryStringQuery) Validate() error { - return nil -} - -func (q *queryStringQuery) Field() string { - return "" -} - -func (q *queryStringQuery) SetField(f string) Query { - return q -} diff --git a/vendor/github.com/blevesearch/bleve/query_string.nex b/vendor/github.com/blevesearch/bleve/query_string.nex deleted file mode 100644 index a53ef24b..00000000 --- a/vendor/github.com/blevesearch/bleve/query_string.nex +++ /dev/null @@ -1,41 +0,0 @@ -/\"((\\\")|(\\\\)|(\\\/)|(\\b)|(\\f)|(\\n)|(\\r)|(\\t)|(\\u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F])|[^\"])*\"/ { - lval.s = yylex.Text()[1:len(yylex.Text())-1] - logDebugTokens("PHRASE - %s", lval.s); - return tPHRASE - } -/\+/ { logDebugTokens("PLUS"); return tPLUS } -/-/ { logDebugTokens("MINUS"); return tMINUS } -/:/ { logDebugTokens("COLON"); return tCOLON } -/\^/ { logDebugTokens("BOOST"); return tBOOST } -/\(/ { logDebugTokens("LPAREN"); return tLPAREN } -/\)/ { logDebugTokens("RPAREN"); return tRPAREN } -/>/ { logDebugTokens("GREATER"); return tGREATER } -/<=~][^\t\n\f\r :^~]*/ { - lval.s = yylex.Text() - logDebugTokens("STRING - %s", lval.s); - return tSTRING - } -// -package bleve - -func logDebugTokens(format string, v ...interface{}) { - if debugLexer { - logger.Printf(format, v...) - } -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/query_string.nn.go b/vendor/github.com/blevesearch/bleve/query_string.nn.go deleted file mode 100644 index 396f8026..00000000 --- a/vendor/github.com/blevesearch/bleve/query_string.nn.go +++ /dev/null @@ -1,1362 +0,0 @@ -package bleve - -import ( - "bufio" - "io" - "strings" -) - -type frame struct { - i int - s string - line, column int -} -type lexer struct { - // The lexer runs in its own goroutine, and communicates via channel 'ch'. - ch chan frame - // We record the level of nesting because the action could return, and a - // subsequent call expects to pick up where it left off. In other words, - // we're simulating a coroutine. - // TODO: Support a channel-based variant that compatible with Go's yacc. - stack []frame - stale bool - - // The 'l' and 'c' fields were added for - // https://github.com/wagerlabs/docker/blob/65694e801a7b80930961d70c69cba9f2465459be/buildfile.nex - // Since then, I introduced the built-in Line() and Column() functions. - l, c int - - // The following line makes it easy for scripts to insert fields in the - // generated code. - // [NEX_END_OF_LEXER_STRUCT] -} - -// newLexerWithInit creates a new lexer object, runs the given callback on it, -// then returns it. -func newLexerWithInit(in io.Reader, initFun func(*lexer)) *lexer { - type dfa struct { - acc []bool // Accepting states. - f []func(rune) int // Transitions. - startf, endf []int // Transitions at start and end of input. - nest []dfa - } - yylex := new(lexer) - if initFun != nil { - initFun(yylex) - } - yylex.ch = make(chan frame) - var scan func(in *bufio.Reader, ch chan frame, family []dfa, line, column int) - scan = func(in *bufio.Reader, ch chan frame, family []dfa, line, column int) { - // Index of DFA and length of highest-precedence match so far. - matchi, matchn := 0, -1 - var buf []rune - n := 0 - checkAccept := func(i int, st int) bool { - // Higher precedence match? DFAs are run in parallel, so matchn is at most len(buf), hence we may omit the length equality check. - if family[i].acc[st] && (matchn < n || matchi > i) { - matchi, matchn = i, n - return true - } - return false - } - var state [][2]int - for i := 0; i < len(family); i++ { - mark := make([]bool, len(family[i].startf)) - // Every DFA starts at state 0. - st := 0 - for { - state = append(state, [2]int{i, st}) - mark[st] = true - // As we're at the start of input, follow all ^ transitions and append to our list of start states. - st = family[i].startf[st] - if -1 == st || mark[st] { - break - } - // We only check for a match after at least one transition. - checkAccept(i, st) - } - } - atEOF := false - for { - if n == len(buf) && !atEOF { - r, _, err := in.ReadRune() - switch err { - case io.EOF: - atEOF = true - case nil: - buf = append(buf, r) - default: - panic(err) - } - } - if !atEOF { - r := buf[n] - n++ - var nextState [][2]int - for _, x := range state { - x[1] = family[x[0]].f[x[1]](r) - if -1 == x[1] { - continue - } - nextState = append(nextState, x) - checkAccept(x[0], x[1]) - } - state = nextState - } else { - dollar: // Handle $. - for _, x := range state { - mark := make([]bool, len(family[x[0]].endf)) - for { - mark[x[1]] = true - x[1] = family[x[0]].endf[x[1]] - if -1 == x[1] || mark[x[1]] { - break - } - if checkAccept(x[0], x[1]) { - // Unlike before, we can break off the search. Now that we're at the end, there's no need to maintain the state of each DFA. - break dollar - } - } - } - state = nil - } - - if state == nil { - lcUpdate := func(r rune) { - if r == '\n' { - line++ - column = 0 - } else { - column++ - } - } - // All DFAs stuck. Return last match if it exists, otherwise advance by one rune and restart all DFAs. - if matchn == -1 { - if len(buf) == 0 { // This can only happen at the end of input. - break - } - lcUpdate(buf[0]) - buf = buf[1:] - } else { - text := string(buf[:matchn]) - buf = buf[matchn:] - matchn = -1 - ch <- frame{matchi, text, line, column} - if len(family[matchi].nest) > 0 { - scan(bufio.NewReader(strings.NewReader(text)), ch, family[matchi].nest, line, column) - } - if atEOF { - break - } - for _, r := range text { - lcUpdate(r) - } - } - n = 0 - for i := 0; i < len(family); i++ { - state = append(state, [2]int{i, 0}) - } - } - } - ch <- frame{-1, "", line, column} - } - go scan(bufio.NewReader(in), yylex.ch, []dfa{ - // \"((\\\")|(\\\\)|(\\\/)|(\\b)|(\\f)|(\\n)|(\\r)|(\\t)|(\\u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F])|[^\"])*\" - {[]bool{false, false, false, true, false, false, false, true, false, false, false, false, false, false, false, false, false, false}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 34: - return 1 - case 47: - return -1 - case 92: - return -1 - case 98: - return -1 - case 102: - return -1 - case 110: - return -1 - case 114: - return -1 - case 116: - return -1 - case 117: - return -1 - } - switch { - case 97 <= r && r <= 102: - return -1 - case 65 <= r && r <= 70: - return -1 - case 48 <= r && r <= 57: - return -1 - } - return -1 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 48 <= r && r <= 57: - return 2 - case 65 <= r && r <= 70: - return 2 - case 97 <= r && r <= 102: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 48 <= r && r <= 57: - return 2 - case 65 <= r && r <= 70: - return 2 - case 97 <= r && r <= 102: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return -1 - case 47: - return -1 - case 92: - return -1 - case 98: - return -1 - case 102: - return -1 - case 110: - return -1 - case 114: - return -1 - case 116: - return -1 - case 117: - return -1 - } - switch { - case 48 <= r && r <= 57: - return -1 - case 65 <= r && r <= 70: - return -1 - case 97 <= r && r <= 102: - return -1 - } - return -1 - }, - func(r rune) int { - switch r { - case 34: - return 7 - case 47: - return 8 - case 92: - return 12 - case 98: - return 13 - case 102: - return 9 - case 110: - return 10 - case 114: - return 5 - case 116: - return 6 - case 117: - return 11 - } - switch { - case 48 <= r && r <= 57: - return 2 - case 65 <= r && r <= 70: - return 2 - case 97 <= r && r <= 102: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - case 48 <= r && r <= 57: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - case 48 <= r && r <= 57: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 48 <= r && r <= 57: - return 2 - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - case 48 <= r && r <= 57: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 65 <= r && r <= 70: - return 2 - case 97 <= r && r <= 102: - return 2 - case 48 <= r && r <= 57: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - case 48 <= r && r <= 57: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 14 - case 102: - return 14 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 97 <= r && r <= 102: - return 14 - case 48 <= r && r <= 57: - return 14 - case 65 <= r && r <= 70: - return 14 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 7 - case 47: - return 8 - case 92: - return 12 - case 98: - return 13 - case 102: - return 9 - case 110: - return 10 - case 114: - return 5 - case 116: - return 6 - case 117: - return 11 - } - switch { - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - case 48 <= r && r <= 57: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 48 <= r && r <= 57: - return 2 - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 15 - case 102: - return 15 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 97 <= r && r <= 102: - return 15 - case 65 <= r && r <= 70: - return 15 - case 48 <= r && r <= 57: - return 15 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 16 - case 102: - return 16 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 48 <= r && r <= 57: - return 16 - case 97 <= r && r <= 102: - return 16 - case 65 <= r && r <= 70: - return 16 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 17 - case 102: - return 17 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 48 <= r && r <= 57: - return 17 - case 97 <= r && r <= 102: - return 17 - case 65 <= r && r <= 70: - return 17 - } - return 2 - }, - func(r rune) int { - switch r { - case 34: - return 3 - case 47: - return 2 - case 92: - return 4 - case 98: - return 2 - case 102: - return 2 - case 110: - return 2 - case 114: - return 2 - case 116: - return 2 - case 117: - return 2 - } - switch { - case 48 <= r && r <= 57: - return 2 - case 97 <= r && r <= 102: - return 2 - case 65 <= r && r <= 70: - return 2 - } - return 2 - }, - }, []int{ /* Start-of-input transitions */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, []int{ /* End-of-input transitions */ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, nil}, - - // \+ - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 43: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 43: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // - - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 45: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // : - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 58: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 58: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // \^ - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 94: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 94: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // \( - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 40: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 40: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // \) - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 41: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 41: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // > - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 62: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 62: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // < - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 60: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 60: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // = - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 61: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 61: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // ~([0-9]|[1-9][0-9]*) - {[]bool{false, false, true, true, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 126: - return 1 - } - switch { - case 48 <= r && r <= 48: - return -1 - case 49 <= r && r <= 57: - return -1 - } - return -1 - }, - func(r rune) int { - switch r { - case 126: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 2 - case 49 <= r && r <= 57: - return 3 - } - return -1 - }, - func(r rune) int { - switch r { - case 126: - return -1 - } - switch { - case 48 <= r && r <= 48: - return -1 - case 49 <= r && r <= 57: - return -1 - } - return -1 - }, - func(r rune) int { - switch r { - case 126: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 4 - case 49 <= r && r <= 57: - return 4 - } - return -1 - }, - func(r rune) int { - switch r { - case 126: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 4 - case 49 <= r && r <= 57: - return 4 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1, -1, -1, -1}, []int{ /* End-of-input transitions */ -1, -1, -1, -1, -1}, nil}, - - // ~ - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 126: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 126: - return -1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // -?([0-9]|[1-9][0-9]*)(\.[0-9][0-9]*)? - {[]bool{false, false, true, true, false, true, true, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 45: - return 1 - case 46: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 2 - case 49 <= r && r <= 57: - return 3 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - case 46: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 2 - case 49 <= r && r <= 57: - return 3 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - case 46: - return 4 - } - switch { - case 48 <= r && r <= 48: - return -1 - case 49 <= r && r <= 57: - return -1 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - case 46: - return 4 - } - switch { - case 48 <= r && r <= 48: - return 5 - case 49 <= r && r <= 57: - return 5 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - case 46: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 6 - case 49 <= r && r <= 57: - return 6 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - case 46: - return 4 - } - switch { - case 48 <= r && r <= 48: - return 5 - case 49 <= r && r <= 57: - return 5 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - case 46: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 7 - case 49 <= r && r <= 57: - return 7 - } - return -1 - }, - func(r rune) int { - switch r { - case 45: - return -1 - case 46: - return -1 - } - switch { - case 48 <= r && r <= 48: - return 7 - case 49 <= r && r <= 57: - return 7 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1, -1, -1, -1, -1, -1, -1}, []int{ /* End-of-input transitions */ -1, -1, -1, -1, -1, -1, -1, -1}, nil}, - - // [ \t\n]+ - {[]bool{false, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 9: - return 1 - case 10: - return 1 - case 32: - return 1 - } - return -1 - }, - func(r rune) int { - switch r { - case 9: - return 1 - case 10: - return 1 - case 32: - return 1 - } - return -1 - }, - }, []int{ /* Start-of-input transitions */ -1, -1}, []int{ /* End-of-input transitions */ -1, -1}, nil}, - - // [^\t\n\f\r :^\+\-><=~][^\t\n\f\r :^~]* - {[]bool{false, true, true}, []func(rune) int{ // Transitions - func(r rune) int { - switch r { - case 9: - return -1 - case 10: - return -1 - case 12: - return -1 - case 13: - return -1 - case 32: - return -1 - case 58: - return -1 - case 60: - return -1 - case 61: - return -1 - case 94: - return -1 - case 126: - return -1 - } - switch { - case 43 <= r && r <= 62: - return -1 - } - return 1 - }, - func(r rune) int { - switch r { - case 9: - return -1 - case 10: - return -1 - case 12: - return -1 - case 13: - return -1 - case 32: - return -1 - case 58: - return -1 - case 60: - return 2 - case 61: - return 2 - case 94: - return -1 - case 126: - return -1 - } - switch { - case 43 <= r && r <= 62: - return 2 - } - return 2 - }, - func(r rune) int { - switch r { - case 9: - return -1 - case 10: - return -1 - case 12: - return -1 - case 13: - return -1 - case 32: - return -1 - case 58: - return -1 - case 60: - return 2 - case 61: - return 2 - case 94: - return -1 - case 126: - return -1 - } - switch { - case 43 <= r && r <= 62: - return 2 - } - return 2 - }, - }, []int{ /* Start-of-input transitions */ -1, -1, -1}, []int{ /* End-of-input transitions */ -1, -1, -1}, nil}, - }, 0, 0) - return yylex -} - -func newLexer(in io.Reader) *lexer { - return newLexerWithInit(in, nil) -} - -// Text returns the matched text. -func (yylex *lexer) Text() string { - return yylex.stack[len(yylex.stack)-1].s -} - -// Line returns the current line number. -// The first line is 0. -func (yylex *lexer) Line() int { - return yylex.stack[len(yylex.stack)-1].line -} - -// Column returns the current column number. -// The first column is 0. -func (yylex *lexer) Column() int { - return yylex.stack[len(yylex.stack)-1].column -} - -func (yylex *lexer) next(lvl int) int { - if lvl == len(yylex.stack) { - l, c := 0, 0 - if lvl > 0 { - l, c = yylex.stack[lvl-1].line, yylex.stack[lvl-1].column - } - yylex.stack = append(yylex.stack, frame{0, "", l, c}) - } - if lvl == len(yylex.stack)-1 { - p := &yylex.stack[lvl] - *p = <-yylex.ch - yylex.stale = false - } else { - yylex.stale = true - } - return yylex.stack[lvl].i -} -func (yylex *lexer) pop() { - yylex.stack = yylex.stack[:len(yylex.stack)-1] -} -func (yylex lexer) Error(e string) { - panic(e) -} - -// Lex runs the lexer. Always returns 0. -// When the -s option is given, this function is not generated; -// instead, the NN_FUN macro runs the lexer. -func (yylex *lexer) Lex(lval *yySymType) int { -OUTER0: - for { - switch yylex.next(0) { - case 0: - { - lval.s = yylex.Text()[1 : len(yylex.Text())-1] - logDebugTokens("PHRASE - %s", lval.s) - return tPHRASE - } - case 1: - { - logDebugTokens("PLUS") - return tPLUS - } - case 2: - { - logDebugTokens("MINUS") - return tMINUS - } - case 3: - { - logDebugTokens("COLON") - return tCOLON - } - case 4: - { - logDebugTokens("BOOST") - return tBOOST - } - case 5: - { - logDebugTokens("LPAREN") - return tLPAREN - } - case 6: - { - logDebugTokens("RPAREN") - return tRPAREN - } - case 7: - { - logDebugTokens("GREATER") - return tGREATER - } - case 8: - { - logDebugTokens("LESS") - return tLESS - } - case 9: - { - logDebugTokens("EQUAL") - return tEQUAL - } - case 10: - { - lval.s = yylex.Text()[1:] - logDebugTokens("TILDENUMBER - %s", lval.s) - return tTILDENUMBER - } - case 11: - { - logDebugTokens("TILDE") - return tTILDE - } - case 12: - { - lval.s = yylex.Text() - logDebugTokens("NUMBER - %s", lval.s) - return tNUMBER - } - case 13: - { - logDebugTokens("WHITESPACE (count=%d)", len(yylex.Text())) /* eat up whitespace */ - } - case 14: - { - lval.s = yylex.Text() - logDebugTokens("STRING - %s", lval.s) - return tSTRING - } - default: - break OUTER0 - } - continue - } - yylex.pop() - - return 0 -} -func logDebugTokens(format string, v ...interface{}) { - if debugLexer { - logger.Printf(format, v...) - } -} diff --git a/vendor/github.com/blevesearch/bleve/query_string.y b/vendor/github.com/blevesearch/bleve/query_string.y deleted file mode 100644 index 54d960f8..00000000 --- a/vendor/github.com/blevesearch/bleve/query_string.y +++ /dev/null @@ -1,217 +0,0 @@ -%{ -package bleve -import "strconv" - -func logDebugGrammar(format string, v ...interface{}) { - if debugParser { - logger.Printf(format, v...) - } -} -%} - -%union { -s string -n int -f float64 -q Query} - -%token tSTRING tPHRASE tPLUS tMINUS tCOLON tBOOST tLPAREN tRPAREN tNUMBER tSTRING tGREATER tLESS -tEQUAL tTILDE tTILDENUMBER - -%type tSTRING -%type tPHRASE -%type tNUMBER -%type tTILDENUMBER -%type searchBase -%type searchSuffix -%type searchPrefix -%type searchMustMustNot -%type searchBoost - -%% - -input: -searchParts { - logDebugGrammar("INPUT") -}; - -searchParts: -searchPart searchParts { - logDebugGrammar("SEARCH PARTS") -} -| -searchPart { - logDebugGrammar("SEARCH PART") -}; - -searchPart: -searchPrefix searchBase searchSuffix { - query := $2 - query.SetBoost($3) - switch($1) { - case queryShould: - yylex.(*lexerWrapper).query.AddShould(query) - case queryMust: - yylex.(*lexerWrapper).query.AddMust(query) - case queryMustNot: - yylex.(*lexerWrapper).query.AddMustNot(query) - } -}; - - -searchPrefix: -/* empty */ { - $$ = queryShould -} -| -searchMustMustNot { - $$ = $1 -} -; - -searchMustMustNot: -tPLUS { - logDebugGrammar("PLUS") - $$ = queryMust -} -| -tMINUS { - logDebugGrammar("MINUS") - $$ = queryMustNot -}; - -searchBase: -tSTRING { - str := $1 - logDebugGrammar("STRING - %s", str) - q := NewMatchQuery(str) - $$ = q -} -| -tSTRING tTILDE { - str := $1 - logDebugGrammar("FUZZY STRING - %s", str) - q := NewMatchQuery(str) - q.SetFuzziness(1) - $$ = q -} -| -tSTRING tCOLON tSTRING tTILDE { - field := $1 - str := $3 - logDebugGrammar("FIELD - %s FUZZY STRING - %s", field, str) - q := NewMatchQuery(str) - q.SetFuzziness(1) - q.SetField(field) - $$ = q -} -| -tSTRING tTILDENUMBER { - str := $1 - fuzziness, _ := strconv.ParseFloat($2, 64) - logDebugGrammar("FUZZY STRING - %s", str) - q := NewMatchQuery(str) - q.SetFuzziness(int(fuzziness)) - $$ = q -} -| -tSTRING tCOLON tSTRING tTILDENUMBER { - field := $1 - str := $3 - fuzziness, _ := strconv.ParseFloat($4, 64) - logDebugGrammar("FIELD - %s FUZZY-%f STRING - %s", field, fuzziness, str) - q := NewMatchQuery(str) - q.SetFuzziness(int(fuzziness)) - q.SetField(field) - $$ = q -} -| -tNUMBER { - str := $1 - logDebugGrammar("STRING - %s", str) - q := NewMatchQuery(str) - $$ = q -} -| -tPHRASE { - phrase := $1 - logDebugGrammar("PHRASE - %s", phrase) - q := NewMatchPhraseQuery(phrase) - $$ = q -} -| -tSTRING tCOLON tSTRING { - field := $1 - str := $3 - logDebugGrammar("FIELD - %s STRING - %s", field, str) - q := NewMatchQuery(str).SetField(field) - $$ = q -} -| -tSTRING tCOLON tNUMBER { - field := $1 - str := $3 - logDebugGrammar("FIELD - %s STRING - %s", field, str) - q := NewMatchQuery(str).SetField(field) - $$ = q -} -| -tSTRING tCOLON tPHRASE { - field := $1 - phrase := $3 - logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) - q := NewMatchPhraseQuery(phrase).SetField(field) - $$ = q -} -| -tSTRING tCOLON tGREATER tNUMBER { - field := $1 - min, _ := strconv.ParseFloat($4, 64) - minInclusive := false - logDebugGrammar("FIELD - GREATER THAN %f", min) - q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil).SetField(field) - $$ = q -} -| -tSTRING tCOLON tGREATER tEQUAL tNUMBER { - field := $1 - min, _ := strconv.ParseFloat($5, 64) - minInclusive := true - logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) - q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil).SetField(field) - $$ = q -} -| -tSTRING tCOLON tLESS tNUMBER { - field := $1 - max, _ := strconv.ParseFloat($4, 64) - maxInclusive := false - logDebugGrammar("FIELD - LESS THAN %f", max) - q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive).SetField(field) - $$ = q -} -| -tSTRING tCOLON tLESS tEQUAL tNUMBER { - field := $1 - max, _ := strconv.ParseFloat($5, 64) - maxInclusive := true - logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) - q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive).SetField(field) - $$ = q -}; - -searchBoost: -tBOOST tNUMBER { - boost, _ := strconv.ParseFloat($2, 64) - $$ = boost - logDebugGrammar("BOOST %f", boost) -}; - -searchSuffix: -/* empty */ { - $$ = 1.0 -} -| -searchBoost { - -}; \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/query_string.y.go b/vendor/github.com/blevesearch/bleve/query_string.y.go deleted file mode 100644 index 469d7163..00000000 --- a/vendor/github.com/blevesearch/bleve/query_string.y.go +++ /dev/null @@ -1,563 +0,0 @@ -package bleve - -import __yyfmt__ "fmt" - -//line query_string.y:2 -import "strconv" - -func logDebugGrammar(format string, v ...interface{}) { - if debugParser { - logger.Printf(format, v...) - } -} - -//line query_string.y:12 -type yySymType struct { - yys int - s string - n int - f float64 - q Query -} - -const tSTRING = 57346 -const tPHRASE = 57347 -const tPLUS = 57348 -const tMINUS = 57349 -const tCOLON = 57350 -const tBOOST = 57351 -const tLPAREN = 57352 -const tRPAREN = 57353 -const tNUMBER = 57354 -const tGREATER = 57355 -const tLESS = 57356 -const tEQUAL = 57357 -const tTILDE = 57358 -const tTILDENUMBER = 57359 - -var yyToknames = []string{ - "tSTRING", - "tPHRASE", - "tPLUS", - "tMINUS", - "tCOLON", - "tBOOST", - "tLPAREN", - "tRPAREN", - "tNUMBER", - "tGREATER", - "tLESS", - "tEQUAL", - "tTILDE", - "tTILDENUMBER", -} -var yyStatenames = []string{} - -const yyEofCode = 1 -const yyErrCode = 2 -const yyMaxDepth = 200 - -//line yacctab:1 -var yyExca = []int{ - -1, 1, - 1, -1, - -2, 0, - -1, 3, - 1, 3, - -2, 5, -} - -const yyNprod = 26 -const yyPrivate = 57344 - -var yyTokenNames []string -var yyStates []string - -const yyLast = 32 - -var yyAct = []int{ - - 17, 25, 26, 20, 22, 32, 31, 29, 16, 18, - 30, 21, 23, 24, 27, 10, 12, 28, 19, 15, - 6, 7, 2, 11, 3, 1, 8, 14, 5, 4, - 13, 9, -} -var yyPact = []int{ - - 14, -1000, -1000, 14, 11, -1000, -1000, -1000, -1000, 10, - -8, -1000, -1000, -1000, -1000, 6, -1000, -1, -1000, -1000, - -15, -1000, -1000, 2, -5, -1000, -1000, -1000, -6, -1000, - -7, -1000, -1000, -} -var yyPgo = []int{ - - 0, 31, 30, 29, 28, 27, 25, 22, 24, -} -var yyR1 = []int{ - - 0, 6, 7, 7, 8, 3, 3, 4, 4, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 5, 2, 2, -} -var yyR2 = []int{ - - 0, 1, 2, 1, 3, 0, 1, 1, 1, 1, - 2, 4, 2, 4, 1, 1, 3, 3, 3, 4, - 5, 4, 5, 2, 0, 1, -} -var yyChk = []int{ - - -1000, -6, -7, -8, -3, -4, 6, 7, -7, -1, - 4, 12, 5, -2, -5, 9, 16, 8, 17, 12, - 4, 12, 5, 13, 14, 16, 17, 12, 15, 12, - 15, 12, 12, -} -var yyDef = []int{ - - 5, -2, 1, -2, 0, 6, 7, 8, 2, 24, - 9, 14, 15, 4, 25, 0, 10, 0, 12, 23, - 16, 17, 18, 0, 0, 11, 13, 19, 0, 21, - 0, 20, 22, -} -var yyTok1 = []int{ - - 1, -} -var yyTok2 = []int{ - - 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, - 12, 13, 14, 15, 16, 17, -} -var yyTok3 = []int{ - 0, -} - -//line yaccpar:1 - -/* parser for yacc output */ - -var yyDebug = 0 - -type yyLexer interface { - Lex(lval *yySymType) int - Error(s string) -} - -const yyFlag = -1000 - -func yyTokname(c int) string { - // 4 is TOKSTART above - if c >= 4 && c-4 < len(yyToknames) { - if yyToknames[c-4] != "" { - return yyToknames[c-4] - } - } - return __yyfmt__.Sprintf("tok-%v", c) -} - -func yyStatname(s int) string { - if s >= 0 && s < len(yyStatenames) { - if yyStatenames[s] != "" { - return yyStatenames[s] - } - } - return __yyfmt__.Sprintf("state-%v", s) -} - -func yylex1(lex yyLexer, lval *yySymType) int { - c := 0 - char := lex.Lex(lval) - if char <= 0 { - c = yyTok1[0] - goto out - } - if char < len(yyTok1) { - c = yyTok1[char] - goto out - } - if char >= yyPrivate { - if char < yyPrivate+len(yyTok2) { - c = yyTok2[char-yyPrivate] - goto out - } - } - for i := 0; i < len(yyTok3); i += 2 { - c = yyTok3[i+0] - if c == char { - c = yyTok3[i+1] - goto out - } - } - -out: - if c == 0 { - c = yyTok2[1] /* unknown char */ - } - if yyDebug >= 3 { - __yyfmt__.Printf("lex %s(%d)\n", yyTokname(c), uint(char)) - } - return c -} - -func yyParse(yylex yyLexer) int { - var yyn int - var yylval yySymType - var yyVAL yySymType - yyS := make([]yySymType, yyMaxDepth) - - Nerrs := 0 /* number of errors */ - Errflag := 0 /* error recovery flag */ - yystate := 0 - yychar := -1 - yyp := -1 - goto yystack - -ret0: - return 0 - -ret1: - return 1 - -yystack: - /* put a state and value onto the stack */ - if yyDebug >= 4 { - __yyfmt__.Printf("char %v in %v\n", yyTokname(yychar), yyStatname(yystate)) - } - - yyp++ - if yyp >= len(yyS) { - nyys := make([]yySymType, len(yyS)*2) - copy(nyys, yyS) - yyS = nyys - } - yyS[yyp] = yyVAL - yyS[yyp].yys = yystate - -yynewstate: - yyn = yyPact[yystate] - if yyn <= yyFlag { - goto yydefault /* simple state */ - } - if yychar < 0 { - yychar = yylex1(yylex, &yylval) - } - yyn += yychar - if yyn < 0 || yyn >= yyLast { - goto yydefault - } - yyn = yyAct[yyn] - if yyChk[yyn] == yychar { /* valid shift */ - yychar = -1 - yyVAL = yylval - yystate = yyn - if Errflag > 0 { - Errflag-- - } - goto yystack - } - -yydefault: - /* default state action */ - yyn = yyDef[yystate] - if yyn == -2 { - if yychar < 0 { - yychar = yylex1(yylex, &yylval) - } - - /* look through exception table */ - xi := 0 - for { - if yyExca[xi+0] == -1 && yyExca[xi+1] == yystate { - break - } - xi += 2 - } - for xi += 2; ; xi += 2 { - yyn = yyExca[xi+0] - if yyn < 0 || yyn == yychar { - break - } - } - yyn = yyExca[xi+1] - if yyn < 0 { - goto ret0 - } - } - if yyn == 0 { - /* error ... attempt to resume parsing */ - switch Errflag { - case 0: /* brand new error */ - yylex.Error("syntax error") - Nerrs++ - if yyDebug >= 1 { - __yyfmt__.Printf("%s", yyStatname(yystate)) - __yyfmt__.Printf(" saw %s\n", yyTokname(yychar)) - } - fallthrough - - case 1, 2: /* incompletely recovered error ... try again */ - Errflag = 3 - - /* find a state where "error" is a legal shift action */ - for yyp >= 0 { - yyn = yyPact[yyS[yyp].yys] + yyErrCode - if yyn >= 0 && yyn < yyLast { - yystate = yyAct[yyn] /* simulate a shift of "error" */ - if yyChk[yystate] == yyErrCode { - goto yystack - } - } - - /* the current p has no shift on "error", pop stack */ - if yyDebug >= 2 { - __yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys) - } - yyp-- - } - /* there is no state on the stack with an error shift ... abort */ - goto ret1 - - case 3: /* no shift yet; clobber input char */ - if yyDebug >= 2 { - __yyfmt__.Printf("error recovery discards %s\n", yyTokname(yychar)) - } - if yychar == yyEofCode { - goto ret1 - } - yychar = -1 - goto yynewstate /* try again in the same state */ - } - } - - /* reduction by production yyn */ - if yyDebug >= 2 { - __yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate)) - } - - yynt := yyn - yypt := yyp - _ = yypt // guard against "declared and not used" - - yyp -= yyR2[yyn] - yyVAL = yyS[yyp+1] - - /* consult goto table to find next state */ - yyn = yyR1[yyn] - yyg := yyPgo[yyn] - yyj := yyg + yyS[yyp].yys + 1 - - if yyj >= yyLast { - yystate = yyAct[yyg] - } else { - yystate = yyAct[yyj] - if yyChk[yystate] != -yyn { - yystate = yyAct[yyg] - } - } - // dummy call; replaced with literal code - switch yynt { - - case 1: - //line query_string.y:34 - { - logDebugGrammar("INPUT") - } - case 2: - //line query_string.y:39 - { - logDebugGrammar("SEARCH PARTS") - } - case 3: - //line query_string.y:43 - { - logDebugGrammar("SEARCH PART") - } - case 4: - //line query_string.y:48 - { - query := yyS[yypt-1].q - query.SetBoost(yyS[yypt-0].f) - switch yyS[yypt-2].n { - case queryShould: - yylex.(*lexerWrapper).query.AddShould(query) - case queryMust: - yylex.(*lexerWrapper).query.AddMust(query) - case queryMustNot: - yylex.(*lexerWrapper).query.AddMustNot(query) - } - } - case 5: - //line query_string.y:63 - { - yyVAL.n = queryShould - } - case 6: - //line query_string.y:67 - { - yyVAL.n = yyS[yypt-0].n - } - case 7: - //line query_string.y:73 - { - logDebugGrammar("PLUS") - yyVAL.n = queryMust - } - case 8: - //line query_string.y:78 - { - logDebugGrammar("MINUS") - yyVAL.n = queryMustNot - } - case 9: - //line query_string.y:84 - { - str := yyS[yypt-0].s - logDebugGrammar("STRING - %s", str) - q := NewMatchQuery(str) - yyVAL.q = q - } - case 10: - //line query_string.y:91 - { - str := yyS[yypt-1].s - logDebugGrammar("FUZZY STRING - %s", str) - q := NewMatchQuery(str) - q.SetFuzziness(1) - yyVAL.q = q - } - case 11: - //line query_string.y:99 - { - field := yyS[yypt-3].s - str := yyS[yypt-1].s - logDebugGrammar("FIELD - %s FUZZY STRING - %s", field, str) - q := NewMatchQuery(str) - q.SetFuzziness(1) - q.SetField(field) - yyVAL.q = q - } - case 12: - //line query_string.y:109 - { - str := yyS[yypt-1].s - fuzziness, _ := strconv.ParseFloat(yyS[yypt-0].s, 64) - logDebugGrammar("FUZZY STRING - %s", str) - q := NewMatchQuery(str) - q.SetFuzziness(int(fuzziness)) - yyVAL.q = q - } - case 13: - //line query_string.y:118 - { - field := yyS[yypt-3].s - str := yyS[yypt-1].s - fuzziness, _ := strconv.ParseFloat(yyS[yypt-0].s, 64) - logDebugGrammar("FIELD - %s FUZZY-%f STRING - %s", field, fuzziness, str) - q := NewMatchQuery(str) - q.SetFuzziness(int(fuzziness)) - q.SetField(field) - yyVAL.q = q - } - case 14: - //line query_string.y:129 - { - str := yyS[yypt-0].s - logDebugGrammar("STRING - %s", str) - q := NewMatchQuery(str) - yyVAL.q = q - } - case 15: - //line query_string.y:136 - { - phrase := yyS[yypt-0].s - logDebugGrammar("PHRASE - %s", phrase) - q := NewMatchPhraseQuery(phrase) - yyVAL.q = q - } - case 16: - //line query_string.y:143 - { - field := yyS[yypt-2].s - str := yyS[yypt-0].s - logDebugGrammar("FIELD - %s STRING - %s", field, str) - q := NewMatchQuery(str).SetField(field) - yyVAL.q = q - } - case 17: - //line query_string.y:151 - { - field := yyS[yypt-2].s - str := yyS[yypt-0].s - logDebugGrammar("FIELD - %s STRING - %s", field, str) - q := NewMatchQuery(str).SetField(field) - yyVAL.q = q - } - case 18: - //line query_string.y:159 - { - field := yyS[yypt-2].s - phrase := yyS[yypt-0].s - logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) - q := NewMatchPhraseQuery(phrase).SetField(field) - yyVAL.q = q - } - case 19: - //line query_string.y:167 - { - field := yyS[yypt-3].s - min, _ := strconv.ParseFloat(yyS[yypt-0].s, 64) - minInclusive := false - logDebugGrammar("FIELD - GREATER THAN %f", min) - q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil).SetField(field) - yyVAL.q = q - } - case 20: - //line query_string.y:176 - { - field := yyS[yypt-4].s - min, _ := strconv.ParseFloat(yyS[yypt-0].s, 64) - minInclusive := true - logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) - q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil).SetField(field) - yyVAL.q = q - } - case 21: - //line query_string.y:185 - { - field := yyS[yypt-3].s - max, _ := strconv.ParseFloat(yyS[yypt-0].s, 64) - maxInclusive := false - logDebugGrammar("FIELD - LESS THAN %f", max) - q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive).SetField(field) - yyVAL.q = q - } - case 22: - //line query_string.y:194 - { - field := yyS[yypt-4].s - max, _ := strconv.ParseFloat(yyS[yypt-0].s, 64) - maxInclusive := true - logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) - q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive).SetField(field) - yyVAL.q = q - } - case 23: - //line query_string.y:204 - { - boost, _ := strconv.ParseFloat(yyS[yypt-0].s, 64) - yyVAL.f = boost - logDebugGrammar("BOOST %f", boost) - } - case 24: - //line query_string.y:211 - { - yyVAL.f = 1.0 - } - case 25: - //line query_string.y:215 - { - - } - } - goto yystack /* stack new state and value */ -} diff --git a/vendor/github.com/blevesearch/bleve/query_string_parser.go b/vendor/github.com/blevesearch/bleve/query_string_parser.go deleted file mode 100644 index 3b035523..00000000 --- a/vendor/github.com/blevesearch/bleve/query_string_parser.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -//go:generate nex query_string.nex -//go:generate sed -i "" -e s/Lexer/lexer/g query_string.nn.go -//go:generate sed -i "" -e s/Newlexer/newLexer/g query_string.nn.go -//go:generate sed -i "" -e s/debuglexer/debugLexer/g query_string.nn.go -//go:generate go fmt query_string.nn.go -//go:generate go tool yacc -o query_string.y.go query_string.y -//go:generate sed -i "" -e 1d query_string.y.go - -package bleve - -import ( - "fmt" - "strings" -) - -var debugParser bool -var debugLexer bool - -func parseQuerySyntax(query string, mapping *IndexMapping) (rq Query, err error) { - lex := newLexerWrapper(newLexer(strings.NewReader(query))) - doParse(lex) - - if len(lex.errs) > 0 { - return nil, fmt.Errorf(strings.Join(lex.errs, "\n")) - } else { - return lex.query, nil - } -} - -func doParse(lex *lexerWrapper) { - defer func() { - r := recover() - if r != nil { - lex.Error("Errors while parsing.") - } - }() - - yyParse(lex) -} - -const ( - queryShould = iota - queryMust - queryMustNot -) - -type lexerWrapper struct { - nex yyLexer - errs []string - query *booleanQuery -} - -func newLexerWrapper(nex yyLexer) *lexerWrapper { - return &lexerWrapper{ - nex: nex, - errs: []string{}, - query: NewBooleanQuery(nil, nil, nil), - } -} - -func (this *lexerWrapper) Lex(lval *yySymType) int { - return this.nex.Lex(lval) -} - -func (this *lexerWrapper) Error(s string) { - this.errs = append(this.errs, s) -} diff --git a/vendor/github.com/blevesearch/bleve/query_string_parser_test.go b/vendor/github.com/blevesearch/bleve/query_string_parser_test.go deleted file mode 100644 index a55efce6..00000000 --- a/vendor/github.com/blevesearch/bleve/query_string_parser_test.go +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "reflect" - "testing" -) - -func TestQuerySyntaxParserValid(t *testing.T) { - tests := []struct { - input string - result Query - mapping *IndexMapping - }{ - { - input: "test", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("test"), - }, - nil), - }, - { - input: `"test phrase 1"`, - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchPhraseQuery("test phrase 1"), - }, - nil), - }, - { - input: "field:test", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("test").SetField("field"), - }, - nil), - }, - { - input: "+field1:test1", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - []Query{ - NewMatchQuery("test1").SetField("field1"), - }, - nil, - nil), - }, - { - input: "-field2:test2", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - nil, - []Query{ - NewMatchQuery("test2").SetField("field2"), - }), - }, - { - input: `field3:"test phrase 2"`, - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchPhraseQuery("test phrase 2").SetField("field3"), - }, - nil), - }, - { - input: `+field4:"test phrase 1"`, - mapping: NewIndexMapping(), - result: NewBooleanQuery( - []Query{ - NewMatchPhraseQuery("test phrase 1").SetField("field4"), - }, - nil, - nil), - }, - { - input: `-field5:"test phrase 2"`, - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - nil, - []Query{ - NewMatchPhraseQuery("test phrase 2").SetField("field5"), - }), - }, - { - input: `+field6:test3 -field7:test4 field8:test5`, - mapping: NewIndexMapping(), - result: NewBooleanQuery( - []Query{ - NewMatchQuery("test3").SetField("field6"), - }, - []Query{ - NewMatchQuery("test5").SetField("field8"), - }, - []Query{ - NewMatchQuery("test4").SetField("field7"), - }), - }, - { - input: "test^3", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("test").SetBoost(3.0), - }, - nil), - }, - { - input: "test^3 other^6", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("test").SetBoost(3.0), - NewMatchQuery("other").SetBoost(6.0), - }, - nil), - }, - { - input: "33", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("33"), - }, - nil), - }, - { - input: "field:33", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("33").SetField("field"), - }, - nil), - }, - { - input: "cat-dog", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("cat-dog"), - }, - nil), - }, - { - input: "watex~", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("watex").SetFuzziness(1), - }, - nil), - }, - { - input: "watex~2", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("watex").SetFuzziness(2), - }, - nil), - }, - { - input: "watex~ 2", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("watex").SetFuzziness(1), - NewMatchQuery("2"), - }, - nil), - }, - { - input: "field:watex~", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("watex").SetFuzziness(1).SetField("field"), - }, - nil), - }, - { - input: "field:watex~2", - mapping: NewIndexMapping(), - result: NewBooleanQuery( - nil, - []Query{ - NewMatchQuery("watex").SetFuzziness(2).SetField("field"), - }, - nil), - }, - } - - for _, test := range tests { - - q, err := parseQuerySyntax(test.input, test.mapping) - if err != nil { - t.Error(err) - } - if !reflect.DeepEqual(q, test.result) { - t.Errorf("Expected %#v, got %#v: for %s", test.result, q, test.input) - } - } -} - -func TestQuerySyntaxParserInvalid(t *testing.T) { - tests := []struct { - input string - }{ - {"^"}, - {"^5"}, - } - - for _, test := range tests { - _, err := parseQuerySyntax(test.input, NewIndexMapping()) - if err == nil { - t.Errorf("expected error, got nil for `%s`", test.input) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/query_term.go b/vendor/github.com/blevesearch/bleve/query_term.go deleted file mode 100644 index df92b454..00000000 --- a/vendor/github.com/blevesearch/bleve/query_term.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -type termQuery struct { - Term string `json:"term"` - FieldVal string `json:"field,omitempty"` - BoostVal float64 `json:"boost,omitempty"` -} - -// NewTermQuery creates a new Query for finding an -// exact term match in the index. -func NewTermQuery(term string) *termQuery { - return &termQuery{ - Term: term, - BoostVal: 1.0, - } -} - -func (q *termQuery) Boost() float64 { - return q.BoostVal -} - -func (q *termQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *termQuery) Field() string { - return q.FieldVal -} - -func (q *termQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *termQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - return searchers.NewTermSearcher(i, q.Term, field, q.BoostVal, explain) -} - -func (q *termQuery) Validate() error { - return nil -} diff --git a/vendor/github.com/blevesearch/bleve/query_test.go b/vendor/github.com/blevesearch/bleve/query_test.go deleted file mode 100644 index df0f5a4d..00000000 --- a/vendor/github.com/blevesearch/bleve/query_test.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "reflect" - "testing" -) - -var minNum = 5.1 -var maxNum = 7.1 -var startDate = "2011-01-01" -var endDate = "2012-01-01" - -func TestParseQuery(t *testing.T) { - tests := []struct { - input []byte - output Query - err error - }{ - { - input: []byte(`{"term":"water","field":"desc"}`), - output: NewTermQuery("water").SetField("desc"), - }, - { - input: []byte(`{"match":"beer","field":"desc"}`), - output: NewMatchQuery("beer").SetField("desc"), - }, - { - input: []byte(`{"match_phrase":"light beer","field":"desc"}`), - output: NewMatchPhraseQuery("light beer").SetField("desc"), - }, - { - input: []byte(`{"must":{"conjuncts": [{"match":"beer","field":"desc"}]},"should":{"disjuncts": [{"match":"water","field":"desc"}],"min":1.0},"must_not":{"disjuncts": [{"match":"devon","field":"desc"}]}}`), - output: NewBooleanQuery( - []Query{NewMatchQuery("beer").SetField("desc")}, - []Query{NewMatchQuery("water").SetField("desc")}, - []Query{NewMatchQuery("devon").SetField("desc")}), - }, - { - input: []byte(`{"terms":[{"term":"watered","field":"desc"},{"term":"down","field":"desc"}]}`), - output: NewPhraseQuery([]string{"watered", "down"}, "desc"), - }, - { - input: []byte(`{"query":"+beer \"light beer\" -devon"}`), - output: NewQueryStringQuery(`+beer "light beer" -devon`), - }, - { - input: []byte(`{"min":5.1,"max":7.1,"field":"desc"}`), - output: NewNumericRangeQuery(&minNum, &maxNum).SetField("desc"), - }, - { - input: []byte(`{"start":"` + startDate + `","end":"` + endDate + `","field":"desc"}`), - output: NewDateRangeQuery(&startDate, &endDate).SetField("desc"), - }, - { - input: []byte(`{"prefix":"budwei","field":"desc"}`), - output: NewPrefixQuery("budwei").SetField("desc"), - }, - { - input: []byte(`{"madeitup":"queryhere"}`), - output: nil, - err: ErrorUnknownQueryType, - }, - } - - for i, test := range tests { - actual, err := ParseQuery(test.input) - if err != nil && test.err == nil { - t.Errorf("error %v for %d", err, i) - } else if test.err != nil { - if !reflect.DeepEqual(err, test.err) { - t.Errorf("expected error: %#v, got: %#v", test.err, err) - } - } - - if !reflect.DeepEqual(test.output, actual) { - t.Errorf("expected: %#v, got: %#v", test.output, actual) - // t.Errorf("expected: %#v, got: %#v", test.output.(*BooleanQuery).Should, actual.(*BooleanQuery).Should) - } - } -} - -func TestSetGetField(t *testing.T) { - tests := []struct { - query Query - field string - }{ - { - query: NewTermQuery("water").SetField("desc"), - field: "desc", - }, - { - query: NewMatchQuery("beer").SetField("desc"), - field: "desc", - }, - { - query: NewMatchPhraseQuery("light beer").SetField("desc"), - field: "desc", - }, - { - query: NewNumericRangeQuery(&minNum, &maxNum).SetField("desc"), - field: "desc", - }, - { - query: NewDateRangeQuery(&startDate, &endDate).SetField("desc"), - field: "desc", - }, - { - query: NewPrefixQuery("budwei").SetField("desc"), - field: "desc", - }, - } - - for _, test := range tests { - query := test.query - if query.Field() != test.field { - t.Errorf("expected field '%s', got '%s'", test.field, query.Field()) - } - } -} - -func TestQueryValidate(t *testing.T) { - tests := []struct { - query Query - err error - }{ - { - query: NewTermQuery("water").SetField("desc"), - err: nil, - }, - { - query: NewMatchQuery("beer").SetField("desc"), - err: nil, - }, - { - query: NewMatchPhraseQuery("light beer").SetField("desc"), - err: nil, - }, - { - query: NewNumericRangeQuery(&minNum, &maxNum).SetField("desc"), - err: nil, - }, - { - query: NewNumericRangeQuery(nil, nil).SetField("desc"), - err: ErrorNumericQueryNoBounds, - }, - { - query: NewDateRangeQuery(&startDate, &endDate).SetField("desc"), - err: nil, - }, - { - query: NewPrefixQuery("budwei").SetField("desc"), - err: nil, - }, - { - query: NewQueryStringQuery(`+beer "light beer" -devon`), - err: nil, - }, - { - query: NewPhraseQuery([]string{"watered", "down"}, "desc"), - err: nil, - }, - { - query: NewPhraseQuery([]string{}, "field"), - err: ErrorPhraseQueryNoTerms, - }, - { - query: NewMatchNoneQuery().SetBoost(25), - err: nil, - }, - { - query: NewMatchAllQuery().SetBoost(25), - err: nil, - }, - { - query: NewBooleanQuery( - []Query{NewMatchQuery("beer").SetField("desc")}, - []Query{NewMatchQuery("water").SetField("desc")}, - []Query{NewMatchQuery("devon").SetField("desc")}), - err: nil, - }, - { - query: NewBooleanQuery( - nil, - nil, - []Query{NewMatchQuery("devon").SetField("desc")}), - err: ErrorBooleanQueryNeedsMustOrShould, - }, - { - query: NewBooleanQuery( - []Query{}, - []Query{}, - []Query{NewMatchQuery("devon").SetField("desc")}), - err: ErrorBooleanQueryNeedsMustOrShould, - }, - { - query: NewBooleanQueryMinShould( - []Query{NewMatchQuery("beer").SetField("desc")}, - []Query{NewMatchQuery("water").SetField("desc")}, - []Query{NewMatchQuery("devon").SetField("desc")}, - 2.0), - err: ErrorDisjunctionFewerThanMinClauses, - }, - } - - for _, test := range tests { - actual := test.query.Validate() - if !reflect.DeepEqual(actual, test.err) { - t.Errorf("expected error: %#v got %#v", test.err, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/query_wildcard.go b/vendor/github.com/blevesearch/bleve/query_wildcard.go deleted file mode 100644 index 32fd3433..00000000 --- a/vendor/github.com/blevesearch/bleve/query_wildcard.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "regexp" - "strings" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/searchers" -) - -var wildcardRegexpReplacer = strings.NewReplacer( - // characters in the wildcard that must - // be escaped in the regexp - "+", `\+`, - "(", `\(`, - ")", `\)`, - "^", `\^`, - "$", `\$`, - ".", `\.`, - "{", `\{`, - "}", `\}`, - "[", `\[`, - "]", `\]`, - `|`, `\|`, - `\`, `\\`, - // wildcard characters - "*", ".*", - "?", ".") - -type wildcardQuery struct { - Wildcard string `json:"wildcard"` - FieldVal string `json:"field,omitempty"` - BoostVal float64 `json:"boost,omitempty"` - compiled *regexp.Regexp -} - -// NewWildcardQuery creates a new Query which finds -// documents containing terms that match the -// specified wildcard. In the wildcard pattern '*' -// will match any sequence of 0 or more characters, -// and '?' will match any single character. -func NewWildcardQuery(wildcard string) *wildcardQuery { - return &wildcardQuery{ - Wildcard: wildcard, - BoostVal: 1.0, - } -} - -func (q *wildcardQuery) Boost() float64 { - return q.BoostVal -} - -func (q *wildcardQuery) SetBoost(b float64) Query { - q.BoostVal = b - return q -} - -func (q *wildcardQuery) Field() string { - return q.FieldVal -} - -func (q *wildcardQuery) SetField(f string) Query { - q.FieldVal = f - return q -} - -func (q *wildcardQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { - field := q.FieldVal - if q.FieldVal == "" { - field = m.DefaultField - } - if q.compiled == nil { - var err error - q.compiled, err = q.convertToRegexp() - if err != nil { - return nil, err - } - } - - return searchers.NewRegexpSearcher(i, q.compiled, field, q.BoostVal, explain) -} - -func (q *wildcardQuery) Validate() error { - var err error - q.compiled, err = q.convertToRegexp() - return err -} - -func (q *wildcardQuery) convertToRegexp() (*regexp.Regexp, error) { - regexpString := "^" + wildcardRegexpReplacer.Replace(q.Wildcard) + "$" - return regexp.Compile(regexpString) -} diff --git a/vendor/github.com/blevesearch/bleve/registry/analyzer.go b/vendor/github.com/blevesearch/bleve/registry/analyzer.go index 8589f59f..340e349d 100644 --- a/vendor/github.com/blevesearch/bleve/registry/analyzer.go +++ b/vendor/github.com/blevesearch/bleve/registry/analyzer.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterAnalyzer(name string, constructor AnalyzerConstructor) { type AnalyzerConstructor func(config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error) type AnalyzerRegistry map[string]AnalyzerConstructor -type AnalyzerCache map[string]*analysis.Analyzer -func (c AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (*analysis.Analyzer, error) { - analyzer, cached := c[name] - if cached { - return analyzer, nil +type AnalyzerCache struct { + *ConcurrentCache +} + +func NewAnalyzerCache() *AnalyzerCache { + return &AnalyzerCache{ + NewConcurrentCache(), } - analyzerConstructor, registered := analyzers[name] +} + +func AnalyzerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := analyzers[name] if !registered { return nil, fmt.Errorf("no analyzer with name or type '%s' registered", name) } - analyzer, err := analyzerConstructor(nil, cache) + analyzer, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building analyzer: %v", err) } - c[name] = analyzer return analyzer, nil } -func (c AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("analyzer named '%s' already defined", name) - } - analyzerConstructor, registered := analyzers[typ] - if !registered { - return nil, fmt.Errorf("no analyzer type '%s' registered", typ) - } - analyzer, err := analyzerConstructor(config, cache) +func (c *AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (*analysis.Analyzer, error) { + item, err := c.ItemNamed(name, cache, AnalyzerBuild) if err != nil { - return nil, fmt.Errorf("error building analyzer: %v", err) + return nil, err } - c[name] = analyzer - return analyzer, nil + return item.(*analysis.Analyzer), nil +} + +func (c *AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error) { + item, err := c.DefineItem(name, typ, config, cache, AnalyzerBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("analyzer named '%s' already defined", name) + } + return nil, err + } + return item.(*analysis.Analyzer), nil } func AnalyzerTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range analyzers { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/byte_array_converter.go b/vendor/github.com/blevesearch/bleve/registry/byte_array_converter.go deleted file mode 100644 index 0adb3cdb..00000000 --- a/vendor/github.com/blevesearch/bleve/registry/byte_array_converter.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package registry - -import ( - "fmt" - - "github.com/blevesearch/bleve/analysis" -) - -func RegisterByteArrayConverter(name string, constructor ByteArrayConverterConstructor) { - _, exists := byteArrayConverters[name] - if exists { - panic(fmt.Errorf("attempted to register duplicate byte array converter named '%s'", name)) - } - byteArrayConverters[name] = constructor -} - -type ByteArrayConverterConstructor func(config map[string]interface{}, cache *Cache) (analysis.ByteArrayConverter, error) -type ByteArrayConverterRegistry map[string]ByteArrayConverterConstructor - -func ByteArrayConverterByName(name string) ByteArrayConverterConstructor { - return byteArrayConverters[name] -} - -func ByteArrayConverterTypesAndInstances() ([]string, []string) { - emptyConfig := map[string]interface{}{} - emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) - for name, cons := range byteArrayConverters { - _, err := cons(emptyConfig, emptyCache) - if err == nil { - instances = append(instances, name) - } else { - types = append(types, name) - } - } - return types, instances -} diff --git a/vendor/github.com/blevesearch/bleve/registry/cache.go b/vendor/github.com/blevesearch/bleve/registry/cache.go new file mode 100644 index 00000000..b0ce852a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/registry/cache.go @@ -0,0 +1,87 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + "sync" +) + +var ErrAlreadyDefined = fmt.Errorf("item already defined") + +type CacheBuild func(name string, config map[string]interface{}, cache *Cache) (interface{}, error) + +type ConcurrentCache struct { + mutex sync.RWMutex + data map[string]interface{} +} + +func NewConcurrentCache() *ConcurrentCache { + return &ConcurrentCache{ + data: make(map[string]interface{}), + } +} + +func (c *ConcurrentCache) ItemNamed(name string, cache *Cache, build CacheBuild) (interface{}, error) { + c.mutex.RLock() + item, cached := c.data[name] + if cached { + c.mutex.RUnlock() + return item, nil + } + // give up read lock + c.mutex.RUnlock() + // try to build it + newItem, err := build(name, nil, cache) + if err != nil { + return nil, err + } + // acquire write lock + c.mutex.Lock() + defer c.mutex.Unlock() + // check again because it could have been created while trading locks + item, cached = c.data[name] + if cached { + return item, nil + } + c.data[name] = newItem + return newItem, nil +} + +func (c *ConcurrentCache) DefineItem(name string, typ string, config map[string]interface{}, cache *Cache, build CacheBuild) (interface{}, error) { + c.mutex.RLock() + _, cached := c.data[name] + if cached { + c.mutex.RUnlock() + return nil, ErrAlreadyDefined + } + // give up read lock so others lookups can proceed + c.mutex.RUnlock() + // really not there, try to build it + newItem, err := build(typ, config, cache) + if err != nil { + return nil, err + } + // now we've built it, acquire lock + c.mutex.Lock() + defer c.mutex.Unlock() + // check again because it could have been created while trading locks + _, cached = c.data[name] + if cached { + return nil, ErrAlreadyDefined + } + c.data[name] = newItem + return newItem, nil +} diff --git a/vendor/github.com/blevesearch/bleve/registry/char_filter.go b/vendor/github.com/blevesearch/bleve/registry/char_filter.go index c063bf37..4696713c 100644 --- a/vendor/github.com/blevesearch/bleve/registry/char_filter.go +++ b/vendor/github.com/blevesearch/bleve/registry/char_filter.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterCharFilter(name string, constructor CharFilterConstructor) { type CharFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) type CharFilterRegistry map[string]CharFilterConstructor -type CharFilterCache map[string]analysis.CharFilter -func (c CharFilterCache) CharFilterNamed(name string, cache *Cache) (analysis.CharFilter, error) { - charFilter, cached := c[name] - if cached { - return charFilter, nil +type CharFilterCache struct { + *ConcurrentCache +} + +func NewCharFilterCache() *CharFilterCache { + return &CharFilterCache{ + NewConcurrentCache(), } - charFilterConstructor, registered := charFilters[name] +} + +func CharFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := charFilters[name] if !registered { return nil, fmt.Errorf("no char filter with name or type '%s' registered", name) } - charFilter, err := charFilterConstructor(nil, cache) + charFilter, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building char filter: %v", err) } - c[name] = charFilter return charFilter, nil } -func (c CharFilterCache) DefineCharFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("char filter named '%s' already defined", name) - } - charFilterConstructor, registered := charFilters[typ] - if !registered { - return nil, fmt.Errorf("no char filter type '%s' registered", typ) - } - charFilter, err := charFilterConstructor(config, cache) +func (c *CharFilterCache) CharFilterNamed(name string, cache *Cache) (analysis.CharFilter, error) { + item, err := c.ItemNamed(name, cache, CharFilterBuild) if err != nil { - return nil, fmt.Errorf("error building char filter: %v", err) + return nil, err } - c[name] = charFilter - return charFilter, nil + return item.(analysis.CharFilter), nil +} + +func (c *CharFilterCache) DefineCharFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) { + item, err := c.DefineItem(name, typ, config, cache, CharFilterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("char filter named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.CharFilter), nil } func CharFilterTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range charFilters { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go b/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go index 29f9c0d8..2cd46e5a 100644 --- a/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go +++ b/vendor/github.com/blevesearch/bleve/registry/datetime_parser.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterDateTimeParser(name string, constructor DateTimeParserConstructor) type DateTimeParserConstructor func(config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) type DateTimeParserRegistry map[string]DateTimeParserConstructor -type DateTimeParserCache map[string]analysis.DateTimeParser -func (c DateTimeParserCache) DateTimeParserNamed(name string, cache *Cache) (analysis.DateTimeParser, error) { - dateTimeParser, cached := c[name] - if cached { - return dateTimeParser, nil +type DateTimeParserCache struct { + *ConcurrentCache +} + +func NewDateTimeParserCache() *DateTimeParserCache { + return &DateTimeParserCache{ + NewConcurrentCache(), } - dateTimeParserConstructor, registered := dateTimeParsers[name] +} + +func DateTimeParserBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := dateTimeParsers[name] if !registered { return nil, fmt.Errorf("no date time parser with name or type '%s' registered", name) } - dateTimeParser, err := dateTimeParserConstructor(nil, cache) - if err != nil { - return nil, fmt.Errorf("error building date time parse: %v", err) - } - c[name] = dateTimeParser - return dateTimeParser, nil -} - -func (c DateTimeParserCache) DefineDateTimeParser(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("date time parser named '%s' already defined", name) - } - dateTimeParserConstructor, registered := dateTimeParsers[typ] - if !registered { - return nil, fmt.Errorf("no date time parser type '%s' registered", typ) - } - dateTimeParser, err := dateTimeParserConstructor(config, cache) + dateTimeParser, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building date time parser: %v", err) } - c[name] = dateTimeParser return dateTimeParser, nil } +func (c *DateTimeParserCache) DateTimeParserNamed(name string, cache *Cache) (analysis.DateTimeParser, error) { + item, err := c.ItemNamed(name, cache, DateTimeParserBuild) + if err != nil { + return nil, err + } + return item.(analysis.DateTimeParser), nil +} + +func (c *DateTimeParserCache) DefineDateTimeParser(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) { + item, err := c.DefineItem(name, typ, config, cache, DateTimeParserBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("date time parser named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.DateTimeParser), nil +} + func DateTimeParserTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range dateTimeParsers { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go b/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go index e9933e3b..d0121d96 100644 --- a/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go +++ b/vendor/github.com/blevesearch/bleve/registry/fragment_formatter.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterFragmentFormatter(name string, constructor FragmentFormatterConstru type FragmentFormatterConstructor func(config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) type FragmentFormatterRegistry map[string]FragmentFormatterConstructor -type FragmentFormatterCache map[string]highlight.FragmentFormatter -func (c FragmentFormatterCache) FragmentFormatterNamed(name string, cache *Cache) (highlight.FragmentFormatter, error) { - fragmentFormatter, cached := c[name] - if cached { - return fragmentFormatter, nil +type FragmentFormatterCache struct { + *ConcurrentCache +} + +func NewFragmentFormatterCache() *FragmentFormatterCache { + return &FragmentFormatterCache{ + NewConcurrentCache(), } - fragmentFormatterConstructor, registered := fragmentFormatters[name] +} + +func FragmentFormatterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := fragmentFormatters[name] if !registered { return nil, fmt.Errorf("no fragment formatter with name or type '%s' registered", name) } - fragmentFormatter, err := fragmentFormatterConstructor(nil, cache) + fragmentFormatter, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building fragment formatter: %v", err) } - c[name] = fragmentFormatter return fragmentFormatter, nil } -func (c FragmentFormatterCache) DefineFragmentFormatter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("fragment formatter named '%s' already defined", name) - } - fragmentFormatterConstructor, registered := fragmentFormatters[typ] - if !registered { - return nil, fmt.Errorf("no fragment formatter type '%s' registered", typ) - } - fragmentFormatter, err := fragmentFormatterConstructor(config, cache) +func (c *FragmentFormatterCache) FragmentFormatterNamed(name string, cache *Cache) (highlight.FragmentFormatter, error) { + item, err := c.ItemNamed(name, cache, FragmentFormatterBuild) if err != nil { - return nil, fmt.Errorf("error building fragment formatter: %v", err) + return nil, err } - c[name] = fragmentFormatter - return fragmentFormatter, nil + return item.(highlight.FragmentFormatter), nil +} + +func (c *FragmentFormatterCache) DefineFragmentFormatter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) { + item, err := c.DefineItem(name, typ, config, cache, FragmentFormatterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("fragment formatter named '%s' already defined", name) + } + return nil, err + } + return item.(highlight.FragmentFormatter), nil } func FragmentFormatterTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range fragmentFormatters { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/fragmenter.go b/vendor/github.com/blevesearch/bleve/registry/fragmenter.go index 91da86a0..18ab2ac0 100644 --- a/vendor/github.com/blevesearch/bleve/registry/fragmenter.go +++ b/vendor/github.com/blevesearch/bleve/registry/fragmenter.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterFragmenter(name string, constructor FragmenterConstructor) { type FragmenterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) type FragmenterRegistry map[string]FragmenterConstructor -type FragmenterCache map[string]highlight.Fragmenter -func (c FragmenterCache) FragmenterNamed(name string, cache *Cache) (highlight.Fragmenter, error) { - fragmenter, cached := c[name] - if cached { - return fragmenter, nil +type FragmenterCache struct { + *ConcurrentCache +} + +func NewFragmenterCache() *FragmenterCache { + return &FragmenterCache{ + NewConcurrentCache(), } - fragmenterConstructor, registered := fragmenters[name] +} + +func FragmenterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := fragmenters[name] if !registered { return nil, fmt.Errorf("no fragmenter with name or type '%s' registered", name) } - fragmenter, err := fragmenterConstructor(nil, cache) + fragmenter, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building fragmenter: %v", err) } - c[name] = fragmenter return fragmenter, nil } -func (c FragmenterCache) DefineFragmenter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("fragmenter named '%s' already defined", name) - } - fragmenterConstructor, registered := fragmenters[typ] - if !registered { - return nil, fmt.Errorf("no fragmenter type '%s' registered", typ) - } - fragmenter, err := fragmenterConstructor(config, cache) +func (c *FragmenterCache) FragmenterNamed(name string, cache *Cache) (highlight.Fragmenter, error) { + item, err := c.ItemNamed(name, cache, FragmenterBuild) if err != nil { - return nil, fmt.Errorf("error building fragmenter: %v", err) + return nil, err } - c[name] = fragmenter - return fragmenter, nil + return item.(highlight.Fragmenter), nil +} + +func (c *FragmenterCache) DefineFragmenter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) { + item, err := c.DefineItem(name, typ, config, cache, FragmenterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("fragmenter named '%s' already defined", name) + } + return nil, err + } + return item.(highlight.Fragmenter), nil } func FragmenterTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range fragmenters { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/highlighter.go b/vendor/github.com/blevesearch/bleve/registry/highlighter.go index 0ead1645..b84219ca 100644 --- a/vendor/github.com/blevesearch/bleve/registry/highlighter.go +++ b/vendor/github.com/blevesearch/bleve/registry/highlighter.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterHighlighter(name string, constructor HighlighterConstructor) { type HighlighterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) type HighlighterRegistry map[string]HighlighterConstructor -type HighlighterCache map[string]highlight.Highlighter -func (c HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) { - highlighter, cached := c[name] - if cached { - return highlighter, nil +type HighlighterCache struct { + *ConcurrentCache +} + +func NewHighlighterCache() *HighlighterCache { + return &HighlighterCache{ + NewConcurrentCache(), } - highlighterConstructor, registered := highlighters[name] +} + +func HighlighterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := highlighters[name] if !registered { return nil, fmt.Errorf("no highlighter with name or type '%s' registered", name) } - highlighter, err := highlighterConstructor(nil, cache) + highlighter, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building highlighter: %v", err) } - c[name] = highlighter return highlighter, nil } -func (c HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("highlighter named '%s' already defined", name) - } - highlighterConstructor, registered := highlighters[typ] - if !registered { - return nil, fmt.Errorf("no highlighter type '%s' registered", typ) - } - highlighter, err := highlighterConstructor(config, cache) +func (c *HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) { + item, err := c.ItemNamed(name, cache, HighlighterBuild) if err != nil { - return nil, fmt.Errorf("error building highlighter: %v", err) + return nil, err } - c[name] = highlighter - return highlighter, nil + return item.(highlight.Highlighter), nil +} + +func (c *HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) { + item, err := c.DefineItem(name, typ, config, cache, HighlighterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("highlighter named '%s' already defined", name) + } + return nil, err + } + return item.(highlight.Highlighter), nil } func HighlighterTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range highlighters { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/index_type.go b/vendor/github.com/blevesearch/bleve/registry/index_type.go new file mode 100644 index 00000000..4da07c82 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/registry/index_type.go @@ -0,0 +1,45 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "fmt" + + "github.com/blevesearch/bleve/index" +) + +func RegisterIndexType(name string, constructor IndexTypeConstructor) { + _, exists := indexTypes[name] + if exists { + panic(fmt.Errorf("attempted to register duplicate index encoding named '%s'", name)) + } + indexTypes[name] = constructor +} + +type IndexTypeConstructor func(storeName string, storeConfig map[string]interface{}, analysisQueue *index.AnalysisQueue) (index.Index, error) +type IndexTypeRegistry map[string]IndexTypeConstructor + +func IndexTypeConstructorByName(name string) IndexTypeConstructor { + return indexTypes[name] +} + +func IndexTypesAndInstances() ([]string, []string) { + var types []string + var instances []string + for name := range stores { + types = append(types, name) + } + return types, instances +} diff --git a/vendor/github.com/blevesearch/bleve/registry/registry.go b/vendor/github.com/blevesearch/bleve/registry/registry.go index 56d6f3ca..a0ea69c8 100644 --- a/vendor/github.com/blevesearch/bleve/registry/registry.go +++ b/vendor/github.com/blevesearch/bleve/registry/registry.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -17,8 +22,7 @@ import ( ) var stores = make(KVStoreRegistry, 0) - -var byteArrayConverters = make(ByteArrayConverterRegistry, 0) +var indexTypes = make(IndexTypeRegistry, 0) // highlight var fragmentFormatters = make(FragmentFormatterRegistry, 0) @@ -34,37 +38,41 @@ var analyzers = make(AnalyzerRegistry, 0) var dateTimeParsers = make(DateTimeParserRegistry, 0) type Cache struct { - CharFilters CharFilterCache - Tokenizers TokenizerCache - TokenMaps TokenMapCache - TokenFilters TokenFilterCache - Analyzers AnalyzerCache - DateTimeParsers DateTimeParserCache - FragmentFormatters FragmentFormatterCache - Fragmenters FragmenterCache - Highlighters HighlighterCache + CharFilters *CharFilterCache + Tokenizers *TokenizerCache + TokenMaps *TokenMapCache + TokenFilters *TokenFilterCache + Analyzers *AnalyzerCache + DateTimeParsers *DateTimeParserCache + FragmentFormatters *FragmentFormatterCache + Fragmenters *FragmenterCache + Highlighters *HighlighterCache } func NewCache() *Cache { return &Cache{ - CharFilters: make(CharFilterCache, 0), - Tokenizers: make(TokenizerCache, 0), - TokenMaps: make(TokenMapCache, 0), - TokenFilters: make(TokenFilterCache, 0), - Analyzers: make(AnalyzerCache, 0), - DateTimeParsers: make(DateTimeParserCache, 0), - FragmentFormatters: make(FragmentFormatterCache, 0), - Fragmenters: make(FragmenterCache, 0), - Highlighters: make(HighlighterCache, 0), + CharFilters: NewCharFilterCache(), + Tokenizers: NewTokenizerCache(), + TokenMaps: NewTokenMapCache(), + TokenFilters: NewTokenFilterCache(), + Analyzers: NewAnalyzerCache(), + DateTimeParsers: NewDateTimeParserCache(), + FragmentFormatters: NewFragmentFormatterCache(), + Fragmenters: NewFragmenterCache(), + Highlighters: NewHighlighterCache(), } } func typeFromConfig(config map[string]interface{}) (string, error) { - typ, ok := config["type"].(string) - if ok { - return typ, nil + prop, ok := config["type"] + if !ok { + return "", fmt.Errorf("'type' property is not defined") } - return "", fmt.Errorf("unable to determine type") + typ, ok := prop.(string) + if !ok { + return "", fmt.Errorf("'type' property must be a string, not %T", prop) + } + return typ, nil } func (c *Cache) CharFilterNamed(name string) (analysis.CharFilter, error) { @@ -86,7 +94,7 @@ func (c *Cache) TokenizerNamed(name string) (analysis.Tokenizer, error) { func (c *Cache) DefineTokenizer(name string, config map[string]interface{}) (analysis.Tokenizer, error) { typ, err := typeFromConfig(config) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot resolve '%s' tokenizer type: %s", name, err) } return c.Tokenizers.DefineTokenizer(name, typ, config, c) } diff --git a/vendor/github.com/blevesearch/bleve/registry/store.go b/vendor/github.com/blevesearch/bleve/registry/store.go index cff7c88e..83187763 100644 --- a/vendor/github.com/blevesearch/bleve/registry/store.go +++ b/vendor/github.com/blevesearch/bleve/registry/store.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -23,7 +28,13 @@ func RegisterKVStore(name string, constructor KVStoreConstructor) { stores[name] = constructor } -type KVStoreConstructor func(config map[string]interface{}) (store.KVStore, error) +// KVStoreConstructor is used to build a KVStore of a specific type when +// specificied by the index configuration. In addition to meeting the +// store.KVStore interface, KVStores must also support this constructor. +// Note that currently the values of config must +// be able to be marshaled and unmarshaled using the encoding/json library (used +// when reading/writing the index metadata file). +type KVStoreConstructor func(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) type KVStoreRegistry map[string]KVStoreConstructor func KVStoreConstructorByName(name string) KVStoreConstructor { @@ -31,16 +42,10 @@ func KVStoreConstructorByName(name string) KVStoreConstructor { } func KVStoreTypesAndInstances() ([]string, []string) { - emptyConfig := map[string]interface{}{} - types := make([]string, 0) - instances := make([]string, 0) - for name, cons := range stores { - _, err := cons(emptyConfig) - if err == nil { - instances = append(instances, name) - } else { - types = append(types, name) - } + var types []string + var instances []string + for name := range stores { + types = append(types, name) } return types, instances } diff --git a/vendor/github.com/blevesearch/bleve/registry/token_filter.go b/vendor/github.com/blevesearch/bleve/registry/token_filter.go index 1e9a9e53..e202e15f 100644 --- a/vendor/github.com/blevesearch/bleve/registry/token_filter.go +++ b/vendor/github.com/blevesearch/bleve/registry/token_filter.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterTokenFilter(name string, constructor TokenFilterConstructor) { type TokenFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) type TokenFilterRegistry map[string]TokenFilterConstructor -type TokenFilterCache map[string]analysis.TokenFilter -func (c TokenFilterCache) TokenFilterNamed(name string, cache *Cache) (analysis.TokenFilter, error) { - tokenFilter, cached := c[name] - if cached { - return tokenFilter, nil +type TokenFilterCache struct { + *ConcurrentCache +} + +func NewTokenFilterCache() *TokenFilterCache { + return &TokenFilterCache{ + NewConcurrentCache(), } - tokenFilterConstructor, registered := tokenFilters[name] +} + +func TokenFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := tokenFilters[name] if !registered { return nil, fmt.Errorf("no token filter with name or type '%s' registered", name) } - tokenFilter, err := tokenFilterConstructor(nil, cache) + tokenFilter, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building token filter: %v", err) } - c[name] = tokenFilter return tokenFilter, nil } -func (c TokenFilterCache) DefineTokenFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("token filter named '%s' already defined", name) - } - tokenFilterConstructor, registered := tokenFilters[typ] - if !registered { - return nil, fmt.Errorf("no token filter type '%s' registered", typ) - } - tokenFilter, err := tokenFilterConstructor(config, cache) +func (c *TokenFilterCache) TokenFilterNamed(name string, cache *Cache) (analysis.TokenFilter, error) { + item, err := c.ItemNamed(name, cache, TokenFilterBuild) if err != nil { - return nil, fmt.Errorf("error building token filter: %v", err) + return nil, err } - c[name] = tokenFilter - return tokenFilter, nil + return item.(analysis.TokenFilter), nil +} + +func (c *TokenFilterCache) DefineTokenFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) { + item, err := c.DefineItem(name, typ, config, cache, TokenFilterBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("token filter named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.TokenFilter), nil } func TokenFilterTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range tokenFilters { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/token_maps.go b/vendor/github.com/blevesearch/bleve/registry/token_maps.go index 03a48aa6..66ca08fd 100644 --- a/vendor/github.com/blevesearch/bleve/registry/token_maps.go +++ b/vendor/github.com/blevesearch/bleve/registry/token_maps.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterTokenMap(name string, constructor TokenMapConstructor) { type TokenMapConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) type TokenMapRegistry map[string]TokenMapConstructor -type TokenMapCache map[string]analysis.TokenMap -func (c TokenMapCache) TokenMapNamed(name string, cache *Cache) (analysis.TokenMap, error) { - tokenMap, cached := c[name] - if cached { - return tokenMap, nil +type TokenMapCache struct { + *ConcurrentCache +} + +func NewTokenMapCache() *TokenMapCache { + return &TokenMapCache{ + NewConcurrentCache(), } - tokenMapConstructor, registered := tokenMaps[name] +} + +func TokenMapBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := tokenMaps[name] if !registered { return nil, fmt.Errorf("no token map with name or type '%s' registered", name) } - tokenMap, err := tokenMapConstructor(nil, cache) + tokenMap, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building token map: %v", err) } - c[name] = tokenMap return tokenMap, nil } -func (c TokenMapCache) DefineTokenMap(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("token map named '%s' already defined", name) - } - tokenMapConstructor, registered := tokenMaps[typ] - if !registered { - return nil, fmt.Errorf("no token map type '%s' registered", typ) - } - tokenMap, err := tokenMapConstructor(config, cache) +func (c *TokenMapCache) TokenMapNamed(name string, cache *Cache) (analysis.TokenMap, error) { + item, err := c.ItemNamed(name, cache, TokenMapBuild) if err != nil { - return nil, fmt.Errorf("error building token map: %v", err) + return nil, err } - c[name] = tokenMap - return tokenMap, nil + return item.(analysis.TokenMap), nil +} + +func (c *TokenMapCache) DefineTokenMap(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) { + item, err := c.DefineItem(name, typ, config, cache, TokenMapBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("token map named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.TokenMap), nil } func TokenMapTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range tokenMaps { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/registry/tokenizer.go b/vendor/github.com/blevesearch/bleve/registry/tokenizer.go index efd8339a..cb9af643 100644 --- a/vendor/github.com/blevesearch/bleve/registry/tokenizer.go +++ b/vendor/github.com/blevesearch/bleve/registry/tokenizer.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package registry @@ -25,47 +30,53 @@ func RegisterTokenizer(name string, constructor TokenizerConstructor) { type TokenizerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) type TokenizerRegistry map[string]TokenizerConstructor -type TokenizerCache map[string]analysis.Tokenizer -func (c TokenizerCache) TokenizerNamed(name string, cache *Cache) (analysis.Tokenizer, error) { - tokenizer, cached := c[name] - if cached { - return tokenizer, nil +type TokenizerCache struct { + *ConcurrentCache +} + +func NewTokenizerCache() *TokenizerCache { + return &TokenizerCache{ + NewConcurrentCache(), } - tokenizerConstructor, registered := tokenizers[name] +} + +func TokenizerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) { + cons, registered := tokenizers[name] if !registered { return nil, fmt.Errorf("no tokenizer with name or type '%s' registered", name) } - tokenizer, err := tokenizerConstructor(nil, cache) + tokenizer, err := cons(config, cache) if err != nil { return nil, fmt.Errorf("error building tokenizer: %v", err) } - c[name] = tokenizer return tokenizer, nil } -func (c TokenizerCache) DefineTokenizer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) { - _, cached := c[name] - if cached { - return nil, fmt.Errorf("tokenizer named '%s' already defined", name) - } - tokenizerConstructor, registered := tokenizers[typ] - if !registered { - return nil, fmt.Errorf("no tokenizer type '%s' registered", typ) - } - tokenizer, err := tokenizerConstructor(config, cache) +func (c *TokenizerCache) TokenizerNamed(name string, cache *Cache) (analysis.Tokenizer, error) { + item, err := c.ItemNamed(name, cache, TokenizerBuild) if err != nil { - return nil, fmt.Errorf("error building tokenizer: %v", err) + return nil, err } - c[name] = tokenizer - return tokenizer, nil + return item.(analysis.Tokenizer), nil +} + +func (c *TokenizerCache) DefineTokenizer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) { + item, err := c.DefineItem(name, typ, config, cache, TokenizerBuild) + if err != nil { + if err == ErrAlreadyDefined { + return nil, fmt.Errorf("tokenizer named '%s' already defined", name) + } + return nil, err + } + return item.(analysis.Tokenizer), nil } func TokenizerTypesAndInstances() ([]string, []string) { emptyConfig := map[string]interface{}{} emptyCache := NewCache() - types := make([]string, 0) - instances := make([]string, 0) + var types []string + var instances []string for name, cons := range tokenizers { _, err := cons(emptyConfig, emptyCache) if err == nil { diff --git a/vendor/github.com/blevesearch/bleve/search.go b/vendor/github.com/blevesearch/bleve/search.go index 1d123af1..98790c9e 100644 --- a/vendor/github.com/blevesearch/bleve/search.go +++ b/vendor/github.com/blevesearch/bleve/search.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package bleve @@ -15,9 +20,16 @@ import ( "time" "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/analysis/datetime/optional" + "github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/query" ) +var cache = registry.NewCache() + +const defaultDateTimeParser = optional.Name + type numericRange struct { Name string `json:"name,omitempty"` Min *float64 `json:"min,omitempty"` @@ -32,19 +44,22 @@ type dateTimeRange struct { endString *string } -func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) { +func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time) { + start = dr.Start if dr.Start.IsZero() && dr.startString != nil { - start, err := dateTimeParser.ParseDateTime(*dr.startString) + s, err := dateTimeParser.ParseDateTime(*dr.startString) if err == nil { - dr.Start = start + start = s } } + end = dr.End if dr.End.IsZero() && dr.endString != nil { - end, err := dateTimeParser.ParseDateTime(*dr.endString) + e, err := dateTimeParser.ParseDateTime(*dr.endString) if err == nil { - dr.End = end + end = e } } + return start, end } func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { @@ -70,6 +85,21 @@ func (dr *dateTimeRange) UnmarshalJSON(input []byte) error { return nil } +func (dr *dateTimeRange) MarshalJSON() ([]byte, error) { + rv := map[string]interface{}{ + "name": dr.Name, + "start": dr.Start, + "end": dr.End, + } + if dr.Start.IsZero() && dr.startString != nil { + rv["start"] = dr.startString + } + if dr.End.IsZero() && dr.endString != nil { + rv["end"] = dr.endString + } + return json.Marshal(rv) +} + // A FacetRequest describes a facet or aggregation // of the result document set you would like to be // built. @@ -80,6 +110,45 @@ type FacetRequest struct { DateTimeRanges []*dateTimeRange `json:"date_ranges,omitempty"` } +func (fr *FacetRequest) Validate() error { + nrCount := len(fr.NumericRanges) + drCount := len(fr.DateTimeRanges) + if nrCount > 0 && drCount > 0 { + return fmt.Errorf("facet can only conain numeric ranges or date ranges, not both") + } + + if nrCount > 0 { + nrNames := map[string]interface{}{} + for _, nr := range fr.NumericRanges { + if _, ok := nrNames[nr.Name]; ok { + return fmt.Errorf("numeric ranges contains duplicate name '%s'", nr.Name) + } + nrNames[nr.Name] = struct{}{} + if nr.Min == nil && nr.Max == nil { + return fmt.Errorf("numeric range query must specify either min, max or both for range name '%s'", nr.Name) + } + } + + } else { + dateTimeParser, err := cache.DateTimeParserNamed(defaultDateTimeParser) + if err != nil { + return err + } + drNames := map[string]interface{}{} + for _, dr := range fr.DateTimeRanges { + if _, ok := drNames[dr.Name]; ok { + return fmt.Errorf("date ranges contains duplicate name '%s'", dr.Name) + } + drNames[dr.Name] = struct{}{} + start, end := dr.ParseDates(dateTimeParser) + if start.IsZero() && end.IsZero() { + return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name) + } + } + } + return nil +} + // NewFacetRequest creates a facet on the specified // field that limits the number of entries to the // specified size. @@ -116,6 +185,16 @@ func (fr *FacetRequest) AddNumericRange(name string, min, max *float64) { // FacetRequest objects for a single query. type FacetsRequest map[string]*FacetRequest +func (fr FacetsRequest) Validate() error { + for _, v := range fr { + err := v.Validate() + if err != nil { + return err + } + } + return nil +} + // HighlightRequest describes how field matches // should be highlighted. type HighlightRequest struct { @@ -152,20 +231,35 @@ func (h *HighlightRequest) AddField(field string) { // Highlight describes optional search result // highlighting. // Fields describes a list of field values which -// should be retrieved for result documents. +// should be retrieved for result documents, provided they +// were stored while indexing. // Facets describe the set of facets to be computed. // Explain triggers inclusion of additional search // result score explanations. +// Sort describes the desired order for the results to be returned. // // A special field named "*" can be used to return all fields. type SearchRequest struct { - Query Query `json:"query"` - Size int `json:"size"` - From int `json:"from"` - Highlight *HighlightRequest `json:"highlight"` - Fields []string `json:"fields"` - Facets FacetsRequest `json:"facets"` - Explain bool `json:"explain"` + Query query.Query `json:"query"` + Size int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight"` + Fields []string `json:"fields"` + Facets FacetsRequest `json:"facets"` + Explain bool `json:"explain"` + Sort search.SortOrder `json:"sort"` + IncludeLocations bool `json:"includeLocations"` +} + +func (r *SearchRequest) Validate() error { + if srq, ok := r.Query.(query.ValidatableQuery); ok { + err := srq.Validate() + if err != nil { + return err + } + } + + return r.Facets.Validate() } // AddFacet adds a FacetRequest to this SearchRequest @@ -176,17 +270,34 @@ func (r *SearchRequest) AddFacet(facetName string, f *FacetRequest) { r.Facets[facetName] = f } +// SortBy changes the request to use the requested sort order +// this form uses the simplified syntax with an array of strings +// each string can either be a field name +// or the magic value _id and _score which refer to the doc id and search score +// any of these values can optionally be prefixed with - to reverse the order +func (r *SearchRequest) SortBy(order []string) { + so := search.ParseSortOrderStrings(order) + r.Sort = so +} + +// SortByCustom changes the request to use the requested sort order +func (r *SearchRequest) SortByCustom(order search.SortOrder) { + r.Sort = order +} + // UnmarshalJSON deserializes a JSON representation of // a SearchRequest func (r *SearchRequest) UnmarshalJSON(input []byte) error { var temp struct { - Q json.RawMessage `json:"query"` - Size int `json:"size"` - From int `json:"from"` - Highlight *HighlightRequest `json:"highlight"` - Fields []string `json:"fields"` - Facets FacetsRequest `json:"facets"` - Explain bool `json:"explain"` + Q json.RawMessage `json:"query"` + Size *int `json:"size"` + From int `json:"from"` + Highlight *HighlightRequest `json:"highlight"` + Fields []string `json:"fields"` + Facets FacetsRequest `json:"facets"` + Explain bool `json:"explain"` + Sort []json.RawMessage `json:"sort"` + IncludeLocations bool `json:"includeLocations"` } err := json.Unmarshal(input, &temp) @@ -194,13 +305,26 @@ func (r *SearchRequest) UnmarshalJSON(input []byte) error { return err } - r.Size = temp.Size + if temp.Size == nil { + r.Size = 10 + } else { + r.Size = *temp.Size + } + if temp.Sort == nil { + r.Sort = search.SortOrder{&search.SortScore{Desc: true}} + } else { + r.Sort, err = search.ParseSortOrderJSON(temp.Sort) + if err != nil { + return err + } + } r.From = temp.From r.Explain = temp.Explain r.Highlight = temp.Highlight r.Fields = temp.Fields r.Facets = temp.Facets - r.Query, err = ParseQuery(temp.Q) + r.IncludeLocations = temp.IncludeLocations + r.Query, err = query.ParseQuery(temp.Q) if err != nil { return err } @@ -219,25 +343,77 @@ func (r *SearchRequest) UnmarshalJSON(input []byte) error { // NewSearchRequest creates a new SearchRequest // for the Query, using default values for all // other search parameters. -func NewSearchRequest(q Query) *SearchRequest { +func NewSearchRequest(q query.Query) *SearchRequest { return NewSearchRequestOptions(q, 10, 0, false) } // NewSearchRequestOptions creates a new SearchRequest // for the Query, with the requested size, from // and explanation search parameters. -func NewSearchRequestOptions(q Query, size, from int, explain bool) *SearchRequest { +// By default results are ordered by score, descending. +func NewSearchRequestOptions(q query.Query, size, from int, explain bool) *SearchRequest { return &SearchRequest{ Query: q, Size: size, From: from, Explain: explain, + Sort: search.SortOrder{&search.SortScore{Desc: true}}, + } +} + +// IndexErrMap tracks errors with the name of the index where it occurred +type IndexErrMap map[string]error + +// MarshalJSON seralizes the error into a string for JSON consumption +func (iem IndexErrMap) MarshalJSON() ([]byte, error) { + tmp := make(map[string]string, len(iem)) + for k, v := range iem { + tmp[k] = v.Error() + } + return json.Marshal(tmp) +} + +func (iem IndexErrMap) UnmarshalJSON(data []byte) error { + var tmp map[string]string + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + for k, v := range tmp { + iem[k] = fmt.Errorf("%s", v) + } + return nil +} + +// SearchStatus is a secion in the SearchResult reporting how many +// underlying indexes were queried, how many were successful/failed +// and a map of any errors that were encountered +type SearchStatus struct { + Total int `json:"total"` + Failed int `json:"failed"` + Successful int `json:"successful"` + Errors IndexErrMap `json:"errors,omitempty"` +} + +// Merge will merge together multiple SearchStatuses during a MultiSearch +func (ss *SearchStatus) Merge(other *SearchStatus) { + ss.Total += other.Total + ss.Failed += other.Failed + ss.Successful += other.Successful + if len(other.Errors) > 0 { + if ss.Errors == nil { + ss.Errors = make(map[string]error) + } + for otherIndex, otherError := range other.Errors { + ss.Errors[otherIndex] = otherError + } } } // A SearchResult describes the results of executing // a SearchRequest. type SearchResult struct { + Status *SearchStatus `json:"status"` Request *SearchRequest `json:"request"` Hits search.DocumentMatchCollection `json:"hits"` Total uint64 `json:"total_hits"` @@ -287,7 +463,9 @@ func (sr *SearchResult) String() string { return rv } +// Merge will merge together multiple SearchResults during a MultiSearch func (sr *SearchResult) Merge(other *SearchResult) { + sr.Status.Merge(other.Status) sr.Hits = append(sr.Hits, other.Hits...) sr.Total += other.Total if other.MaxScore > sr.MaxScore { diff --git a/vendor/github.com/blevesearch/bleve/search/collector.go b/vendor/github.com/blevesearch/bleve/search/collector.go index 86e901af..cba4829d 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector.go +++ b/vendor/github.com/blevesearch/bleve/search/collector.go @@ -1,20 +1,29 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package search import ( "time" + + "github.com/blevesearch/bleve/index" + + "golang.org/x/net/context" ) type Collector interface { - Collect(searcher Searcher) error + Collect(ctx context.Context, searcher Searcher, reader index.IndexReader) error Results() DocumentMatchCollection Total() uint64 MaxScore() float64 diff --git a/vendor/github.com/blevesearch/bleve/search/collector/heap.go b/vendor/github.com/blevesearch/bleve/search/collector/heap.go new file mode 100644 index 00000000..19f3a059 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/collector/heap.go @@ -0,0 +1,91 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "container/heap" + + "github.com/blevesearch/bleve/search" +) + +type collectStoreHeap struct { + heap search.DocumentMatchCollection + compare collectorCompare +} + +func newStoreHeap(cap int, compare collectorCompare) *collectStoreHeap { + rv := &collectStoreHeap{ + heap: make(search.DocumentMatchCollection, 0, cap), + compare: compare, + } + heap.Init(rv) + return rv +} + +func (c *collectStoreHeap) Add(doc *search.DocumentMatch) { + heap.Push(c, doc) +} + +func (c *collectStoreHeap) RemoveLast() *search.DocumentMatch { + return heap.Pop(c).(*search.DocumentMatch) +} + +func (c *collectStoreHeap) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) { + count := c.Len() + size := count - skip + if size <= 0 { + return make(search.DocumentMatchCollection, 0), nil + } + rv := make(search.DocumentMatchCollection, size) + for count > 0 { + count-- + + if count >= skip { + size-- + doc := heap.Pop(c).(*search.DocumentMatch) + rv[size] = doc + err := fixup(doc) + if err != nil { + return nil, err + } + } + } + return rv, nil +} + +// heap interface implementation + +func (c *collectStoreHeap) Len() int { + return len(c.heap) +} + +func (c *collectStoreHeap) Less(i, j int) bool { + so := c.compare(c.heap[i], c.heap[j]) + return -so < 0 +} + +func (c *collectStoreHeap) Swap(i, j int) { + c.heap[i], c.heap[j] = c.heap[j], c.heap[i] +} + +func (c *collectStoreHeap) Push(x interface{}) { + c.heap = append(c.heap, x.(*search.DocumentMatch)) +} + +func (c *collectStoreHeap) Pop() interface{} { + var rv *search.DocumentMatch + rv, c.heap = c.heap[len(c.heap)-1], c.heap[:len(c.heap)-1] + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/collector/list.go b/vendor/github.com/blevesearch/bleve/search/collector/list.go new file mode 100644 index 00000000..51b47b12 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/collector/list.go @@ -0,0 +1,78 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "container/list" + + "github.com/blevesearch/bleve/search" +) + +type collectStoreList struct { + results *list.List + compare collectorCompare +} + +func newStoreList(cap int, compare collectorCompare) *collectStoreList { + rv := &collectStoreList{ + results: list.New(), + compare: compare, + } + + return rv +} + +func (c *collectStoreList) Add(doc *search.DocumentMatch) { + for e := c.results.Front(); e != nil; e = e.Next() { + curr := e.Value.(*search.DocumentMatch) + if c.compare(doc, curr) >= 0 { + c.results.InsertBefore(doc, e) + return + } + } + // if we got to the end, we still have to add it + c.results.PushBack(doc) +} + +func (c *collectStoreList) RemoveLast() *search.DocumentMatch { + return c.results.Remove(c.results.Front()).(*search.DocumentMatch) +} + +func (c *collectStoreList) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) { + if c.results.Len()-skip > 0 { + rv := make(search.DocumentMatchCollection, c.results.Len()-skip) + i := 0 + skipped := 0 + for e := c.results.Back(); e != nil; e = e.Prev() { + if skipped < skip { + skipped++ + continue + } + + rv[i] = e.Value.(*search.DocumentMatch) + err := fixup(rv[i]) + if err != nil { + return nil, err + } + i++ + } + return rv, nil + } + return search.DocumentMatchCollection{}, nil +} + +func (c *collectStoreList) Len() int { + return c.results.Len() +} diff --git a/vendor/github.com/blevesearch/bleve/search/collector/slice.go b/vendor/github.com/blevesearch/bleve/search/collector/slice.go new file mode 100644 index 00000000..b061643e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/collector/slice.go @@ -0,0 +1,68 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import "github.com/blevesearch/bleve/search" + +type collectStoreSlice struct { + slice search.DocumentMatchCollection + compare collectorCompare +} + +func newStoreSlice(cap int, compare collectorCompare) *collectStoreSlice { + rv := &collectStoreSlice{ + slice: make(search.DocumentMatchCollection, 0, cap), + compare: compare, + } + return rv +} + +func (c *collectStoreSlice) Add(doc *search.DocumentMatch) { + // find where to insert, starting at end (lowest) + i := len(c.slice) + for ; i > 0; i-- { + cmp := c.compare(doc, c.slice[i-1]) + if cmp >= 0 { + break + } + } + // insert at i + c.slice = append(c.slice, nil) + copy(c.slice[i+1:], c.slice[i:]) + c.slice[i] = doc +} + +func (c *collectStoreSlice) RemoveLast() *search.DocumentMatch { + var rv *search.DocumentMatch + rv, c.slice = c.slice[len(c.slice)-1], c.slice[:len(c.slice)-1] + return rv +} + +func (c *collectStoreSlice) Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) { + for i := skip; i < len(c.slice); i++ { + err := fixup(c.slice[i]) + if err != nil { + return nil, err + } + } + if skip <= len(c.slice) { + return c.slice[skip:], nil + } + return search.DocumentMatchCollection{}, nil +} + +func (c *collectStoreSlice) Len() int { + return len(c.slice) +} diff --git a/vendor/github.com/blevesearch/bleve/search/collector/topn.go b/vendor/github.com/blevesearch/bleve/search/collector/topn.go new file mode 100644 index 00000000..946ca3ec --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/collector/topn.go @@ -0,0 +1,278 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "time" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "golang.org/x/net/context" +) + +// PreAllocSizeSkipCap will cap preallocation to this amount when +// size+skip exceeds this value +var PreAllocSizeSkipCap = 1000 + +type collectorCompare func(i, j *search.DocumentMatch) int + +type collectorFixup func(d *search.DocumentMatch) error + +// TopNCollector collects the top N hits, optionally skipping some results +type TopNCollector struct { + size int + skip int + total uint64 + maxScore float64 + took time.Duration + sort search.SortOrder + results search.DocumentMatchCollection + facetsBuilder *search.FacetsBuilder + + store *collectStoreHeap + + needDocIds bool + neededFields []string + cachedScoring []bool + cachedDesc []bool + + lowestMatchOutsideResults *search.DocumentMatch +} + +// CheckDoneEvery controls how frequently we check the context deadline +const CheckDoneEvery = uint64(1024) + +// NewTopNCollector builds a collector to find the top 'size' hits +// skipping over the first 'skip' hits +// ordering hits by the provided sort order +func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector { + hc := &TopNCollector{size: size, skip: skip, sort: sort} + + // pre-allocate space on the store to avoid reslicing + // unless the size + skip is too large, then cap it + // everything should still work, just reslices as necessary + backingSize := size + skip + 1 + if size+skip > PreAllocSizeSkipCap { + backingSize = PreAllocSizeSkipCap + 1 + } + + hc.store = newStoreHeap(backingSize, func(i, j *search.DocumentMatch) int { + return hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j) + }) + + // these lookups traverse an interface, so do once up-front + if sort.RequiresDocID() { + hc.needDocIds = true + } + hc.neededFields = sort.RequiredFields() + hc.cachedScoring = sort.CacheIsScore() + hc.cachedDesc = sort.CacheDescending() + + return hc +} + +// Collect goes to the index to find the matching documents +func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { + startTime := time.Now() + var err error + var next *search.DocumentMatch + + // pre-allocate enough space in the DocumentMatchPool + // unless the size + skip is too large, then cap it + // everything should still work, just allocates DocumentMatches on demand + backingSize := hc.size + hc.skip + 1 + if hc.size+hc.skip > PreAllocSizeSkipCap { + backingSize = PreAllocSizeSkipCap + 1 + } + searchContext := &search.SearchContext{ + DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)), + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + next, err = searcher.Next(searchContext) + } + for err == nil && next != nil { + if hc.total%CheckDoneEvery == 0 { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + } + + err = hc.collectSingle(searchContext, reader, next) + if err != nil { + break + } + + next, err = searcher.Next(searchContext) + } + // compute search duration + hc.took = time.Since(startTime) + if err != nil { + return err + } + // finalize actual results + err = hc.finalizeResults(reader) + if err != nil { + return err + } + return nil +} + +var sortByScoreOpt = []string{"_score"} + +func (hc *TopNCollector) collectSingle(ctx *search.SearchContext, reader index.IndexReader, d *search.DocumentMatch) error { + var err error + + // visit field terms for features that require it (sort, facets) + if len(hc.neededFields) > 0 { + err = hc.visitFieldTerms(reader, d) + if err != nil { + return err + } + } + + // increment total hits + hc.total++ + d.HitNumber = hc.total + + // update max score + if d.Score > hc.maxScore { + hc.maxScore = d.Score + } + + // see if we need to load ID (at this early stage, for example to sort on it) + if hc.needDocIds { + d.ID, err = reader.ExternalID(d.IndexInternalID) + if err != nil { + return err + } + } + + // compute this hits sort value + if len(hc.sort) == 1 && hc.cachedScoring[0] { + d.Sort = sortByScoreOpt + } else { + hc.sort.Value(d) + } + + // optimization, we track lowest sorting hit already removed from heap + // with this one comparison, we can avoid all heap operations if + // this hit would have been added and then immediately removed + if hc.lowestMatchOutsideResults != nil { + cmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, d, hc.lowestMatchOutsideResults) + if cmp >= 0 { + // this hit can't possibly be in the result set, so avoid heap ops + ctx.DocumentMatchPool.Put(d) + return nil + } + } + + hc.store.Add(d) + if hc.store.Len() > hc.size+hc.skip { + removed := hc.store.RemoveLast() + if hc.lowestMatchOutsideResults == nil { + hc.lowestMatchOutsideResults = removed + } else { + cmp := hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, removed, hc.lowestMatchOutsideResults) + if cmp < 0 { + tmp := hc.lowestMatchOutsideResults + hc.lowestMatchOutsideResults = removed + ctx.DocumentMatchPool.Put(tmp) + } + } + } + + return nil +} + +// visitFieldTerms is responsible for visiting the field terms of the +// search hit, and passing visited terms to the sort and facet builder +func (hc *TopNCollector) visitFieldTerms(reader index.IndexReader, d *search.DocumentMatch) error { + if hc.facetsBuilder != nil { + hc.facetsBuilder.StartDoc() + } + + err := reader.DocumentVisitFieldTerms(d.IndexInternalID, hc.neededFields, func(field string, term []byte) { + if hc.facetsBuilder != nil { + hc.facetsBuilder.UpdateVisitor(field, term) + } + hc.sort.UpdateVisitor(field, term) + }) + + if hc.facetsBuilder != nil { + hc.facetsBuilder.EndDoc() + } + + return err +} + +// SetFacetsBuilder registers a facet builder for this collector +func (hc *TopNCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { + hc.facetsBuilder = facetsBuilder + hc.neededFields = append(hc.neededFields, hc.facetsBuilder.RequiredFields()...) +} + +// finalizeResults starts with the heap containing the final top size+skip +// it now throws away the results to be skipped +// and does final doc id lookup (if necessary) +func (hc *TopNCollector) finalizeResults(r index.IndexReader) error { + var err error + hc.results, err = hc.store.Final(hc.skip, func(doc *search.DocumentMatch) error { + if doc.ID == "" { + // look up the id since we need it for lookup + var err error + doc.ID, err = r.ExternalID(doc.IndexInternalID) + if err != nil { + return err + } + } + return nil + }) + + return err +} + +// Results returns the collected hits +func (hc *TopNCollector) Results() search.DocumentMatchCollection { + return hc.results +} + +// Total returns the total number of hits +func (hc *TopNCollector) Total() uint64 { + return hc.total +} + +// MaxScore returns the maximum score seen across all the hits +func (hc *TopNCollector) MaxScore() float64 { + return hc.maxScore +} + +// Took returns the time spent collecting hits +func (hc *TopNCollector) Took() time.Duration { + return hc.took +} + +// FacetResults returns the computed facets results +func (hc *TopNCollector) FacetResults() search.FacetResults { + if hc.facetsBuilder != nil { + return hc.facetsBuilder.Results() + } + return search.FacetResults{} +} diff --git a/vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score.go b/vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score.go deleted file mode 100644 index 5a31c3e9..00000000 --- a/vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package collectors - -import ( - "container/list" - "time" - - "github.com/blevesearch/bleve/search" -) - -type TopScoreCollector struct { - k int - skip int - results *list.List - took time.Duration - maxScore float64 - total uint64 - facetsBuilder *search.FacetsBuilder -} - -func NewTopScorerCollector(k int) *TopScoreCollector { - return &TopScoreCollector{ - k: k, - skip: 0, - results: list.New(), - } -} - -func NewTopScorerSkipCollector(k, skip int) *TopScoreCollector { - return &TopScoreCollector{ - k: k, - skip: skip, - results: list.New(), - } -} - -func (tksc *TopScoreCollector) Total() uint64 { - return tksc.total -} - -func (tksc *TopScoreCollector) MaxScore() float64 { - return tksc.maxScore -} - -func (tksc *TopScoreCollector) Took() time.Duration { - return tksc.took -} - -func (tksc *TopScoreCollector) Collect(searcher search.Searcher) error { - startTime := time.Now() - next, err := searcher.Next() - for err == nil && next != nil { - tksc.collectSingle(next) - if tksc.facetsBuilder != nil { - err = tksc.facetsBuilder.Update(next) - if err != nil { - break - } - } - next, err = searcher.Next() - } - // compute search duration - tksc.took = time.Since(startTime) - if err != nil { - return err - } - return nil -} - -func (tksc *TopScoreCollector) collectSingle(dm *search.DocumentMatch) { - // increment total hits - tksc.total++ - - // update max score - if dm.Score > tksc.maxScore { - tksc.maxScore = dm.Score - } - - for e := tksc.results.Front(); e != nil; e = e.Next() { - curr := e.Value.(*search.DocumentMatch) - if dm.Score < curr.Score { - - tksc.results.InsertBefore(dm, e) - // if we just made the list too long - if tksc.results.Len() > (tksc.k + tksc.skip) { - // remove the head - tksc.results.Remove(tksc.results.Front()) - } - return - } - } - // if we got to the end, we still have to add it - tksc.results.PushBack(dm) - if tksc.results.Len() > (tksc.k + tksc.skip) { - // remove the head - tksc.results.Remove(tksc.results.Front()) - } -} - -func (tksc *TopScoreCollector) Results() search.DocumentMatchCollection { - if tksc.results.Len()-tksc.skip > 0 { - rv := make(search.DocumentMatchCollection, tksc.results.Len()-tksc.skip) - i := 0 - skipped := 0 - for e := tksc.results.Back(); e != nil; e = e.Prev() { - if skipped < tksc.skip { - skipped++ - continue - } - rv[i] = e.Value.(*search.DocumentMatch) - i++ - } - return rv - } - return search.DocumentMatchCollection{} -} - -func (tksc *TopScoreCollector) SetFacetsBuilder(facetsBuilder *search.FacetsBuilder) { - tksc.facetsBuilder = facetsBuilder -} - -func (tksc *TopScoreCollector) FacetResults() search.FacetResults { - if tksc.facetsBuilder != nil { - return tksc.facetsBuilder.Results() - } - return search.FacetResults{} -} diff --git a/vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score_test.go b/vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score_test.go deleted file mode 100644 index ee21140b..00000000 --- a/vendor/github.com/blevesearch/bleve/search/collectors/collector_top_score_test.go +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package collectors - -import ( - "math/rand" - "strconv" - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestTop10Scores(t *testing.T) { - - // a stub search with more than 10 matches - // the top-10 scores are > 10 - // everything else is less than 10 - searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", - Score: 11, - }, - &search.DocumentMatch{ - ID: "b", - Score: 9, - }, - &search.DocumentMatch{ - ID: "c", - Score: 11, - }, - &search.DocumentMatch{ - ID: "d", - Score: 9, - }, - &search.DocumentMatch{ - ID: "e", - Score: 11, - }, - &search.DocumentMatch{ - ID: "f", - Score: 9, - }, - &search.DocumentMatch{ - ID: "g", - Score: 11, - }, - &search.DocumentMatch{ - ID: "h", - Score: 9, - }, - &search.DocumentMatch{ - ID: "i", - Score: 11, - }, - &search.DocumentMatch{ - ID: "j", - Score: 11, - }, - &search.DocumentMatch{ - ID: "k", - Score: 11, - }, - &search.DocumentMatch{ - ID: "l", - Score: 99, - }, - &search.DocumentMatch{ - ID: "m", - Score: 11, - }, - &search.DocumentMatch{ - ID: "n", - Score: 11, - }, - }, - } - - collector := NewTopScorerCollector(10) - err := collector.Collect(searcher) - if err != nil { - t.Fatal(err) - } - - maxScore := collector.MaxScore() - if maxScore != 99.0 { - t.Errorf("expected max score 99.0, got %f", maxScore) - } - - total := collector.Total() - if total != 14 { - t.Errorf("expected 14 total results, got %d", total) - } - - results := collector.Results() - - if len(results) != 10 { - t.Fatalf("expected 10 results, got %d", len(results)) - } - - if results[0].ID != "l" { - t.Errorf("expected first result to have ID 'l', got %s", results[0].ID) - } - - if results[0].Score != 99.0 { - t.Errorf("expected highest score to be 99.0, got %f", results[0].Score) - } - - minScore := 1000.0 - for _, result := range results { - if result.Score < minScore { - minScore = result.Score - } - } - - if minScore < 10 { - t.Errorf("expected minimum score to be higher than 10, got %f", minScore) - } -} - -func TestTop10ScoresSkip10(t *testing.T) { - - // a stub search with more than 10 matches - // the top-10 scores are > 10 - // everything else is less than 10 - searcher := &stubSearcher{ - matches: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", - Score: 11, - }, - &search.DocumentMatch{ - ID: "b", - Score: 9.5, - }, - &search.DocumentMatch{ - ID: "c", - Score: 11, - }, - &search.DocumentMatch{ - ID: "d", - Score: 9, - }, - &search.DocumentMatch{ - ID: "e", - Score: 11, - }, - &search.DocumentMatch{ - ID: "f", - Score: 9, - }, - &search.DocumentMatch{ - ID: "g", - Score: 11, - }, - &search.DocumentMatch{ - ID: "h", - Score: 9, - }, - &search.DocumentMatch{ - ID: "i", - Score: 11, - }, - &search.DocumentMatch{ - ID: "j", - Score: 11, - }, - &search.DocumentMatch{ - ID: "k", - Score: 11, - }, - &search.DocumentMatch{ - ID: "l", - Score: 99, - }, - &search.DocumentMatch{ - ID: "m", - Score: 11, - }, - &search.DocumentMatch{ - ID: "n", - Score: 11, - }, - }, - } - - collector := NewTopScorerSkipCollector(10, 10) - err := collector.Collect(searcher) - if err != nil { - t.Fatal(err) - } - - maxScore := collector.MaxScore() - if maxScore != 99.0 { - t.Errorf("expected max score 99.0, got %f", maxScore) - } - - total := collector.Total() - if total != 14 { - t.Errorf("expected 14 total results, got %d", total) - } - - results := collector.Results() - - if len(results) != 4 { - t.Fatalf("expected 4 results, got %d", len(results)) - } - - if results[0].ID != "b" { - t.Errorf("expected first result to have ID 'b', got %s", results[0].ID) - } - - if results[0].Score != 9.5 { - t.Errorf("expected highest score to be 9.5ß, got %f", results[0].Score) - } -} - -func BenchmarkTop10of100000Scores(b *testing.B) { - - matches := make(search.DocumentMatchCollection, 0, 100000) - for i := 0; i < 100000; i++ { - matches = append(matches, &search.DocumentMatch{ - ID: strconv.Itoa(i), - Score: rand.Float64(), - }) - } - searcher := &stubSearcher{ - matches: matches, - } - - collector := NewTopScorerCollector(10) - b.ResetTimer() - - err := collector.Collect(searcher) - if err != nil { - b.Fatal(err) - } - res := collector.Results() - for _, dm := range res { - b.Logf("%s - %f\n", dm.ID, dm.Score) - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/collectors/search_test.go b/vendor/github.com/blevesearch/bleve/search/collectors/search_test.go deleted file mode 100644 index dde33ea2..00000000 --- a/vendor/github.com/blevesearch/bleve/search/collectors/search_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package collectors - -import ( - "github.com/blevesearch/bleve/search" -) - -type stubSearcher struct { - index int - matches search.DocumentMatchCollection -} - -func (ss *stubSearcher) Next() (*search.DocumentMatch, error) { - if ss.index < len(ss.matches) { - rv := ss.matches[ss.index] - ss.index++ - return rv, nil - } - return nil, nil -} - -func (ss *stubSearcher) Advance(ID string) (*search.DocumentMatch, error) { - - for ss.index < len(ss.matches) && ss.matches[ss.index].ID < ID { - ss.index++ - } - if ss.index < len(ss.matches) { - rv := ss.matches[ss.index] - ss.index++ - return rv, nil - } - return nil, nil -} - -func (ss *stubSearcher) Close() error { - return nil -} - -func (ss *stubSearcher) Weight() float64 { - return 0.0 -} - -func (ss *stubSearcher) SetQueryNorm(float64) { -} - -func (ss *stubSearcher) Count() uint64 { - return uint64(len(ss.matches)) -} - -func (ss *stubSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/explanation.go b/vendor/github.com/blevesearch/bleve/search/explanation.go index fe2d5870..766367d7 100644 --- a/vendor/github.com/blevesearch/bleve/search/explanation.go +++ b/vendor/github.com/blevesearch/bleve/search/explanation.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package search diff --git a/vendor/github.com/blevesearch/bleve/search/facets/benchmark_data.txt b/vendor/github.com/blevesearch/bleve/search/facet/benchmark_data.txt similarity index 100% rename from vendor/github.com/blevesearch/bleve/search/facets/benchmark_data.txt rename to vendor/github.com/blevesearch/bleve/search/facet/benchmark_data.txt diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go new file mode 100644 index 00000000..8657a553 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go @@ -0,0 +1,134 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package facet + +import ( + "sort" + "time" + + "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/search" +) + +type dateTimeRange struct { + start time.Time + end time.Time +} + +type DateTimeFacetBuilder struct { + size int + field string + termsCount map[string]int + total int + missing int + ranges map[string]*dateTimeRange + sawValue bool +} + +func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { + return &DateTimeFacetBuilder{ + size: size, + field: field, + termsCount: make(map[string]int), + ranges: make(map[string]*dateTimeRange, 0), + } +} + +func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) { + r := dateTimeRange{ + start: start, + end: end, + } + fb.ranges[name] = &r +} + +func (fb *DateTimeFacetBuilder) Field() string { + return fb.field +} + +func (fb *DateTimeFacetBuilder) UpdateVisitor(field string, term []byte) { + if field == fb.field { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + t := time.Unix(0, i64) + + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ + } + } + } + } + } +} + +func (fb *DateTimeFacetBuilder) StartDoc() { + fb.sawValue = false +} + +func (fb *DateTimeFacetBuilder) EndDoc() { + if !fb.sawValue { + fb.missing++ + } +} + +func (fb *DateTimeFacetBuilder) Result() *search.FacetResult { + rv := search.FacetResult{ + Field: fb.field, + Total: fb.total, + Missing: fb.missing, + } + + rv.DateRanges = make([]*search.DateRangeFacet, 0, len(fb.termsCount)) + + for term, count := range fb.termsCount { + dateRange := fb.ranges[term] + tf := &search.DateRangeFacet{ + Name: term, + Count: count, + } + if !dateRange.start.IsZero() { + start := dateRange.start.Format(time.RFC3339Nano) + tf.Start = &start + } + if !dateRange.end.IsZero() { + end := dateRange.end.Format(time.RFC3339Nano) + tf.End = &end + } + rv.DateRanges = append(rv.DateRanges, tf) + } + + sort.Sort(rv.DateRanges) + + // we now have the list of the top N facets + if fb.size < len(rv.DateRanges) { + rv.DateRanges = rv.DateRanges[:fb.size] + } + + notOther := 0 + for _, nr := range rv.DateRanges { + notOther += nr.Count + } + rv.Other = fb.total - notOther + + return &rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_numeric.go b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go similarity index 51% rename from vendor/github.com/blevesearch/bleve/search/facets/facet_builder_numeric.go rename to vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go index f5acfb0f..2ab5f278 100644 --- a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_numeric.go +++ b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go @@ -1,19 +1,23 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package facets +package facet import ( "sort" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric_util" + "github.com/blevesearch/bleve/numeric" "github.com/blevesearch/bleve/search" ) @@ -29,6 +33,7 @@ type NumericFacetBuilder struct { total int missing int ranges map[string]*numericRange + sawValue bool } func NewNumericFacetBuilder(field string, size int) *NumericFacetBuilder { @@ -48,36 +53,39 @@ func (fb *NumericFacetBuilder) AddRange(name string, min, max *float64) { fb.ranges[name] = &r } -func (fb *NumericFacetBuilder) Update(ft index.FieldTerms) { - terms, ok := ft[fb.field] - if ok { - for _, term := range terms { - // only consider the values which are shifted 0 - prefixCoded := numeric_util.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - f64 := numeric_util.Int64ToFloat64(i64) +func (fb *NumericFacetBuilder) Field() string { + return fb.field +} - // look at each of the ranges for a match - for rangeName, r := range fb.ranges { +func (fb *NumericFacetBuilder) UpdateVisitor(field string, term []byte) { + if field == fb.field { + fb.sawValue = true + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + f64 := numeric.Int64ToFloat64(i64) - if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) { - - existingCount, existed := fb.termsCount[rangeName] - if existed { - fb.termsCount[rangeName] = existingCount + 1 - } else { - fb.termsCount[rangeName] = 1 - } - fb.total++ - } + // look at each of the ranges for a match + for rangeName, r := range fb.ranges { + if (r.min == nil || f64 >= *r.min) && (r.max == nil || f64 < *r.max) { + fb.termsCount[rangeName] = fb.termsCount[rangeName] + 1 + fb.total++ } } } } - } else { + } +} + +func (fb *NumericFacetBuilder) StartDoc() { + fb.sawValue = false +} + +func (fb *NumericFacetBuilder) EndDoc() { + if !fb.sawValue { fb.missing++ } } diff --git a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_terms.go b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go similarity index 52% rename from vendor/github.com/blevesearch/bleve/search/facets/facet_builder_terms.go rename to vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go index 35c56f22..a41e475a 100644 --- a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_terms.go +++ b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go @@ -1,18 +1,22 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package facets +package facet import ( "sort" - "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) @@ -22,6 +26,7 @@ type TermsFacetBuilder struct { termsCount map[string]int total int missing int + sawValue bool } func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder { @@ -32,19 +37,24 @@ func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder { } } -func (fb *TermsFacetBuilder) Update(ft index.FieldTerms) { - terms, ok := ft[fb.field] - if ok { - for _, term := range terms { - existingCount, existed := fb.termsCount[term] - if existed { - fb.termsCount[term] = existingCount + 1 - } else { - fb.termsCount[term] = 1 - } - fb.total++ - } - } else { +func (fb *TermsFacetBuilder) Field() string { + return fb.field +} + +func (fb *TermsFacetBuilder) UpdateVisitor(field string, term []byte) { + if field == fb.field { + fb.sawValue = true + fb.termsCount[string(term)] = fb.termsCount[string(term)] + 1 + fb.total++ + } +} + +func (fb *TermsFacetBuilder) StartDoc() { + fb.sawValue = false +} + +func (fb *TermsFacetBuilder) EndDoc() { + if !fb.sawValue { fb.missing++ } } diff --git a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_datetime.go deleted file mode 100644 index 3c761291..00000000 --- a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_datetime.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package facets - -import ( - "container/list" - "time" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric_util" - "github.com/blevesearch/bleve/search" -) - -type dateTimeRange struct { - start time.Time - end time.Time -} - -type DateTimeFacetBuilder struct { - size int - field string - termsCount map[string]int - total int - missing int - ranges map[string]*dateTimeRange -} - -func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { - return &DateTimeFacetBuilder{ - size: size, - field: field, - termsCount: make(map[string]int), - ranges: make(map[string]*dateTimeRange, 0), - } -} - -func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) { - r := dateTimeRange{ - start: start, - end: end, - } - fb.ranges[name] = &r -} - -func (fb *DateTimeFacetBuilder) Update(ft index.FieldTerms) { - terms, ok := ft[fb.field] - if ok { - for _, term := range terms { - // only consider the values which are shifted 0 - prefixCoded := numeric_util.PrefixCoded(term) - shift, err := prefixCoded.Shift() - if err == nil && shift == 0 { - i64, err := prefixCoded.Int64() - if err == nil { - t := time.Unix(0, i64) - - // look at each of the ranges for a match - for rangeName, r := range fb.ranges { - - if (r.start.IsZero() || t.After(r.start) || t.Equal(r.start)) && (r.end.IsZero() || t.Before(r.end)) { - - existingCount, existed := fb.termsCount[rangeName] - if existed { - fb.termsCount[rangeName] = existingCount + 1 - } else { - fb.termsCount[rangeName] = 1 - } - fb.total++ - } - } - } - } - } - } else { - fb.missing++ - } -} - -func (fb *DateTimeFacetBuilder) Result() *search.FacetResult { - rv := search.FacetResult{ - Field: fb.field, - Total: fb.total, - Missing: fb.missing, - } - - // FIXME better implementation needed here this is quick and dirty - topN := list.New() - - // walk entries and find top N -OUTER: - for term, count := range fb.termsCount { - dateRange := fb.ranges[term] - tf := &search.DateRangeFacet{ - Name: term, - Count: count, - } - if !dateRange.start.IsZero() { - start := dateRange.start.Format(time.RFC3339Nano) - tf.Start = &start - } - if !dateRange.end.IsZero() { - end := dateRange.end.Format(time.RFC3339Nano) - tf.End = &end - } - - for e := topN.Front(); e != nil; e = e.Next() { - curr := e.Value.(*search.DateRangeFacet) - if tf.Count < curr.Count { - - topN.InsertBefore(tf, e) - // if we just made the list too long - if topN.Len() > fb.size { - // remove the head - topN.Remove(topN.Front()) - } - continue OUTER - } - } - // if we got to the end, we still have to add it - topN.PushBack(tf) - if topN.Len() > fb.size { - // remove the head - topN.Remove(topN.Front()) - } - - } - - // we now have the list of the top N facets - rv.DateRanges = make([]*search.DateRangeFacet, topN.Len()) - i := 0 - notOther := 0 - for e := topN.Back(); e != nil; e = e.Prev() { - rv.DateRanges[i] = e.Value.(*search.DateRangeFacet) - i++ - notOther += e.Value.(*search.DateRangeFacet).Count - } - rv.Other = fb.total - notOther - - return &rv -} diff --git a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_numeric_test.go b/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_numeric_test.go deleted file mode 100644 index 6831438b..00000000 --- a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_numeric_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package facets - -import ( - "strconv" - "testing" - - "github.com/blevesearch/bleve/index" - nu "github.com/blevesearch/bleve/numeric_util" -) - -var pcodedvalues []nu.PrefixCoded - -func init() { - pcodedvalues = []nu.PrefixCoded{nu.PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, nu.PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f}, nu.PrefixCoded{0x20, 0x0, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7a, 0x1d, 0xa}, nu.PrefixCoded{0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x1, 0x16, 0x9, 0x4a, 0x7b}} -} - -func BenchmarkNumericFacet10(b *testing.B) { - numericFacetN(b, 10) -} - -func BenchmarkNumericFacet100(b *testing.B) { - numericFacetN(b, 100) -} - -func BenchmarkNumericFacet1000(b *testing.B) { - numericFacetN(b, 1000) -} - -func numericFacetN(b *testing.B, numTerms int) { - field := "test" - nfb := NewNumericFacetBuilder(field, numTerms) - min, max := 0.0, 9999999998.0 - - for i := 0; i <= numTerms; i++ { - max++ - min-- - - nfb.AddRange("rangename"+strconv.Itoa(i), &min, &max) - - for _, pv := range pcodedvalues { - nfb.Update(index.FieldTerms{field: []string{string(pv)}}) - } - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - nfb.Result() - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_terms_test.go b/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_terms_test.go deleted file mode 100644 index b5e0f647..00000000 --- a/vendor/github.com/blevesearch/bleve/search/facets/facet_builder_terms_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package facets - -import ( - "io/ioutil" - "regexp" - "testing" - - "github.com/blevesearch/bleve/index" -) - -var terms []string - -func init() { - wsRegexp := regexp.MustCompile(`\W+`) - input, err := ioutil.ReadFile("benchmark_data.txt") - if err != nil { - panic(err) - } - terms = wsRegexp.Split(string(input), -1) -} - -func BenchmarkTermsFacet10(b *testing.B) { - termsFacetN(b, 10) -} - -func BenchmarkTermsFacet100(b *testing.B) { - termsFacetN(b, 100) -} - -func BenchmarkTermsFacet1000(b *testing.B) { - termsFacetN(b, 1000) -} - -func BenchmarkTermsFacet10000(b *testing.B) { - termsFacetN(b, 10000) -} - -// func BenchmarkTermsFacet100000(b *testing.B) { -// termsFacetN(b, 100000) -// } - -func termsFacetN(b *testing.B, numTerms int) { - field := "test" - termsLen := len(terms) - tfb := NewTermsFacetBuilder(field, 3) - i := 0 - for len(tfb.termsCount) < numTerms && i <= termsLen { - j := i % termsLen - term := terms[j] - tfb.Update(index.FieldTerms{field: []string{term}}) - i++ - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - tfb.Result() - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/facets_builder.go b/vendor/github.com/blevesearch/bleve/search/facets_builder.go index 28bc5e5e..05e27041 100644 --- a/vendor/github.com/blevesearch/bleve/search/facets_builder.go +++ b/vendor/github.com/blevesearch/bleve/search/facets_builder.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package search @@ -16,13 +21,18 @@ import ( ) type FacetBuilder interface { - Update(index.FieldTerms) + StartDoc() + UpdateVisitor(field string, term []byte) + EndDoc() + Result() *FacetResult + Field() string } type FacetsBuilder struct { indexReader index.IndexReader facets map[string]FacetBuilder + fields []string } func NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder { @@ -34,17 +44,29 @@ func NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder { func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { fb.facets[name] = facetBuilder + fb.fields = append(fb.fields, facetBuilder.Field()) } -func (fb *FacetsBuilder) Update(docMatch *DocumentMatch) error { - fieldTerms, err := fb.indexReader.DocumentFieldTerms(docMatch.ID) - if err != nil { - return err - } +func (fb *FacetsBuilder) RequiredFields() []string { + return fb.fields +} + +func (fb *FacetsBuilder) StartDoc() { for _, facetBuilder := range fb.facets { - facetBuilder.Update(fieldTerms) + facetBuilder.StartDoc() + } +} + +func (fb *FacetsBuilder) EndDoc() { + for _, facetBuilder := range fb.facets { + facetBuilder.EndDoc() + } +} + +func (fb *FacetsBuilder) UpdateVisitor(field string, term []byte) { + for _, facetBuilder := range fb.facets { + facetBuilder.UpdateVisitor(field, term) } - return nil } type TermFacet struct { @@ -66,9 +88,14 @@ func (tf TermFacets) Add(termFacet *TermFacet) TermFacets { return tf } -func (tf TermFacets) Len() int { return len(tf) } -func (tf TermFacets) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] } -func (tf TermFacets) Less(i, j int) bool { return tf[i].Count > tf[j].Count } +func (tf TermFacets) Len() int { return len(tf) } +func (tf TermFacets) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] } +func (tf TermFacets) Less(i, j int) bool { + if tf[i].Count == tf[j].Count { + return tf[i].Term < tf[j].Term + } + return tf[i].Count > tf[j].Count +} type NumericRangeFacet struct { Name string `json:"name"` @@ -77,11 +104,34 @@ type NumericRangeFacet struct { Count int `json:"count"` } +func (nrf *NumericRangeFacet) Same(other *NumericRangeFacet) bool { + if nrf.Min == nil && other.Min != nil { + return false + } + if nrf.Min != nil && other.Min == nil { + return false + } + if nrf.Min != nil && other.Min != nil && *nrf.Min != *other.Min { + return false + } + if nrf.Max == nil && other.Max != nil { + return false + } + if nrf.Max != nil && other.Max == nil { + return false + } + if nrf.Max != nil && other.Max != nil && *nrf.Max != *other.Max { + return false + } + + return true +} + type NumericRangeFacets []*NumericRangeFacet func (nrf NumericRangeFacets) Add(numericRangeFacet *NumericRangeFacet) NumericRangeFacets { for _, existingNr := range nrf { - if numericRangeFacet.Min == existingNr.Min && numericRangeFacet.Max == existingNr.Max { + if numericRangeFacet.Same(existingNr) { existingNr.Count += numericRangeFacet.Count return nrf } @@ -91,9 +141,14 @@ func (nrf NumericRangeFacets) Add(numericRangeFacet *NumericRangeFacet) NumericR return nrf } -func (nrf NumericRangeFacets) Len() int { return len(nrf) } -func (nrf NumericRangeFacets) Swap(i, j int) { nrf[i], nrf[j] = nrf[j], nrf[i] } -func (nrf NumericRangeFacets) Less(i, j int) bool { return nrf[i].Count > nrf[j].Count } +func (nrf NumericRangeFacets) Len() int { return len(nrf) } +func (nrf NumericRangeFacets) Swap(i, j int) { nrf[i], nrf[j] = nrf[j], nrf[i] } +func (nrf NumericRangeFacets) Less(i, j int) bool { + if nrf[i].Count == nrf[j].Count { + return nrf[i].Name < nrf[j].Name + } + return nrf[i].Count > nrf[j].Count +} type DateRangeFacet struct { Name string `json:"name"` @@ -102,11 +157,34 @@ type DateRangeFacet struct { Count int `json:"count"` } +func (drf *DateRangeFacet) Same(other *DateRangeFacet) bool { + if drf.Start == nil && other.Start != nil { + return false + } + if drf.Start != nil && other.Start == nil { + return false + } + if drf.Start != nil && other.Start != nil && *drf.Start != *other.Start { + return false + } + if drf.End == nil && other.End != nil { + return false + } + if drf.End != nil && other.End == nil { + return false + } + if drf.End != nil && other.End != nil && *drf.End != *other.End { + return false + } + + return true +} + type DateRangeFacets []*DateRangeFacet func (drf DateRangeFacets) Add(dateRangeFacet *DateRangeFacet) DateRangeFacets { for _, existingDr := range drf { - if dateRangeFacet.Start == existingDr.Start && dateRangeFacet.End == existingDr.End { + if dateRangeFacet.Same(existingDr) { existingDr.Count += dateRangeFacet.Count return drf } @@ -116,9 +194,14 @@ func (drf DateRangeFacets) Add(dateRangeFacet *DateRangeFacet) DateRangeFacets { return drf } -func (drf DateRangeFacets) Len() int { return len(drf) } -func (drf DateRangeFacets) Swap(i, j int) { drf[i], drf[j] = drf[j], drf[i] } -func (drf DateRangeFacets) Less(i, j int) bool { return drf[i].Count > drf[j].Count } +func (drf DateRangeFacets) Len() int { return len(drf) } +func (drf DateRangeFacets) Swap(i, j int) { drf[i], drf[j] = drf[j], drf[i] } +func (drf DateRangeFacets) Less(i, j int) bool { + if drf[i].Count == drf[j].Count { + return drf[i].Name < drf[j].Name + } + return drf[i].Count > drf[j].Count +} type FacetResult struct { Field string `json:"field"` diff --git a/vendor/github.com/blevesearch/bleve/search/facets_builder_test.go b/vendor/github.com/blevesearch/bleve/search/facets_builder_test.go deleted file mode 100644 index 7e7b66ef..00000000 --- a/vendor/github.com/blevesearch/bleve/search/facets_builder_test.go +++ /dev/null @@ -1,301 +0,0 @@ -package search - -import ( - "reflect" - "testing" -) - -func TestTermFacetResultsMerge(t *testing.T) { - - fr1 := &FacetResult{ - Field: "type", - Total: 100, - Missing: 25, - Other: 25, - Terms: []*TermFacet{ - &TermFacet{ - Term: "blog", - Count: 25, - }, - &TermFacet{ - Term: "comment", - Count: 24, - }, - &TermFacet{ - Term: "feedback", - Count: 1, - }, - }, - } - fr1Only := &FacetResult{ - Field: "category", - Total: 97, - Missing: 22, - Other: 15, - Terms: []*TermFacet{ - &TermFacet{ - Term: "clothing", - Count: 35, - }, - &TermFacet{ - Term: "electronics", - Count: 25, - }, - }, - } - frs1 := FacetResults{ - "types": fr1, - "categories": fr1Only, - } - - fr2 := &FacetResult{ - Field: "type", - Total: 100, - Missing: 25, - Other: 25, - Terms: []*TermFacet{ - &TermFacet{ - Term: "blog", - Count: 25, - }, - &TermFacet{ - Term: "comment", - Count: 22, - }, - &TermFacet{ - Term: "flag", - Count: 3, - }, - }, - } - frs2 := FacetResults{ - "types": fr2, - } - - expectedFr := &FacetResult{ - Field: "type", - Total: 200, - Missing: 50, - Other: 51, - Terms: []*TermFacet{ - &TermFacet{ - Term: "blog", - Count: 50, - }, - &TermFacet{ - Term: "comment", - Count: 46, - }, - &TermFacet{ - Term: "flag", - Count: 3, - }, - }, - } - expectedFrs := FacetResults{ - "types": expectedFr, - "categories": fr1Only, - } - - frs1.Merge(frs2) - frs1.Fixup("types", 3) - if !reflect.DeepEqual(frs1, expectedFrs) { - t.Errorf("expected %v, got %v", expectedFrs, frs1) - } -} - -func TestNumericFacetResultsMerge(t *testing.T) { - - lowmed := 3.0 - medhi := 6.0 - hihigher := 9.0 - - fr1 := &FacetResult{ - Field: "rating", - Total: 100, - Missing: 25, - Other: 25, - NumericRanges: []*NumericRangeFacet{ - &NumericRangeFacet{ - Name: "low", - Max: &lowmed, - Count: 25, - }, - &NumericRangeFacet{ - Name: "med", - Count: 24, - Max: &lowmed, - Min: &medhi, - }, - &NumericRangeFacet{ - Name: "hi", - Count: 1, - Min: &medhi, - Max: &hihigher, - }, - }, - } - frs1 := FacetResults{ - "ratings": fr1, - } - - fr2 := &FacetResult{ - Field: "rating", - Total: 100, - Missing: 25, - Other: 25, - NumericRanges: []*NumericRangeFacet{ - &NumericRangeFacet{ - Name: "low", - Max: &lowmed, - Count: 25, - }, - &NumericRangeFacet{ - Name: "med", - Max: &lowmed, - Min: &medhi, - Count: 22, - }, - &NumericRangeFacet{ - Name: "highest", - Min: &hihigher, - Count: 3, - }, - }, - } - frs2 := FacetResults{ - "ratings": fr2, - } - - expectedFr := &FacetResult{ - Field: "rating", - Total: 200, - Missing: 50, - Other: 51, - NumericRanges: []*NumericRangeFacet{ - &NumericRangeFacet{ - Name: "low", - Count: 50, - Max: &lowmed, - }, - &NumericRangeFacet{ - Name: "med", - Max: &lowmed, - Min: &medhi, - Count: 46, - }, - &NumericRangeFacet{ - Name: "highest", - Min: &hihigher, - Count: 3, - }, - }, - } - expectedFrs := FacetResults{ - "ratings": expectedFr, - } - - frs1.Merge(frs2) - frs1.Fixup("ratings", 3) - if !reflect.DeepEqual(frs1, expectedFrs) { - t.Errorf("expected %#v, got %#v", expectedFrs, frs1) - } -} - -func TestDateFacetResultsMerge(t *testing.T) { - - lowmed := "2010-01-01" - medhi := "2011-01-01" - hihigher := "2012-01-01" - - fr1 := &FacetResult{ - Field: "birthday", - Total: 100, - Missing: 25, - Other: 25, - DateRanges: []*DateRangeFacet{ - &DateRangeFacet{ - Name: "low", - End: &lowmed, - Count: 25, - }, - &DateRangeFacet{ - Name: "med", - Count: 24, - Start: &lowmed, - End: &medhi, - }, - &DateRangeFacet{ - Name: "hi", - Count: 1, - Start: &medhi, - End: &hihigher, - }, - }, - } - frs1 := FacetResults{ - "birthdays": fr1, - } - - fr2 := &FacetResult{ - Field: "birthday", - Total: 100, - Missing: 25, - Other: 25, - DateRanges: []*DateRangeFacet{ - &DateRangeFacet{ - Name: "low", - End: &lowmed, - Count: 25, - }, - &DateRangeFacet{ - Name: "med", - Start: &lowmed, - End: &medhi, - Count: 22, - }, - &DateRangeFacet{ - Name: "highest", - Start: &hihigher, - Count: 3, - }, - }, - } - frs2 := FacetResults{ - "birthdays": fr2, - } - - expectedFr := &FacetResult{ - Field: "birthday", - Total: 200, - Missing: 50, - Other: 51, - DateRanges: []*DateRangeFacet{ - &DateRangeFacet{ - Name: "low", - Count: 50, - End: &lowmed, - }, - &DateRangeFacet{ - Name: "med", - Start: &lowmed, - End: &medhi, - Count: 46, - }, - &DateRangeFacet{ - Name: "highest", - Start: &hihigher, - Count: 3, - }, - }, - } - expectedFrs := FacetResults{ - "birthdays": expectedFr, - } - - frs1.Merge(frs2) - frs1.Fixup("birthdays", 3) - if !reflect.DeepEqual(frs1, expectedFrs) { - t.Errorf("expected %#v, got %#v", expectedFrs, frs1) - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/html/fragment_formatter_html.go b/vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go similarity index 67% rename from vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/html/fragment_formatter_html.go rename to vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go index cd774fbe..8154e790 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/html/fragment_formatter_html.go +++ b/vendor/github.com/blevesearch/bleve/search/highlight/format/html/html.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package html @@ -16,8 +21,8 @@ import ( const Name = "html" -const defaultHTMLHighlightBefore = "" -const defaultHTMLHighlightAfter = "" +const defaultHTMLHighlightBefore = "" +const defaultHTMLHighlightAfter = "" type FragmentFormatter struct { before string @@ -38,6 +43,10 @@ func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations h if termLocation == nil { continue } + // make sure the array positions match + if !termLocation.ArrayPositions.Equals(f.ArrayPositions) { + continue + } if termLocation.Start < curr { continue } diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/ansi/fragment_formatter_ansi.go b/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/ansi/fragment_formatter_ansi.go deleted file mode 100644 index f5c74c5c..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/ansi/fragment_formatter_ansi.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package ansi - -import ( - "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/bleve/search/highlight" -) - -const Name = "ansi" - -const DefaultAnsiHighlight = BgYellow - -type FragmentFormatter struct { - color string -} - -func NewFragmentFormatter(color string) *FragmentFormatter { - return &FragmentFormatter{ - color: color, - } -} - -func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string { - rv := "" - curr := f.Start - for _, termLocation := range orderedTermLocations { - if termLocation == nil { - continue - } - if termLocation.Start < curr { - continue - } - if termLocation.End > f.End { - break - } - // add the stuff before this location - rv += string(f.Orig[curr:termLocation.Start]) - // add the color - rv += a.color - // add the term itself - rv += string(f.Orig[termLocation.Start:termLocation.End]) - // reset the color - rv += Reset - // update current - curr = termLocation.End - } - // add any remaining text after the last token - rv += string(f.Orig[curr:f.End]) - - return rv -} - -// ANSI color control escape sequences. -// Shamelessly copied from https://github.com/sqp/godock/blob/master/libs/log/colors.go -const ( - Reset = "\x1b[0m" - Bright = "\x1b[1m" - Dim = "\x1b[2m" - Underscore = "\x1b[4m" - Blink = "\x1b[5m" - Reverse = "\x1b[7m" - Hidden = "\x1b[8m" - FgBlack = "\x1b[30m" - FgRed = "\x1b[31m" - FgGreen = "\x1b[32m" - FgYellow = "\x1b[33m" - FgBlue = "\x1b[34m" - FgMagenta = "\x1b[35m" - FgCyan = "\x1b[36m" - FgWhite = "\x1b[37m" - BgBlack = "\x1b[40m" - BgRed = "\x1b[41m" - BgGreen = "\x1b[42m" - BgYellow = "\x1b[43m" - BgBlue = "\x1b[44m" - BgMagenta = "\x1b[45m" - BgCyan = "\x1b[46m" - BgWhite = "\x1b[47m" -) - -func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) { - color := DefaultAnsiHighlight - colorVal, ok := config["color"].(string) - if ok { - color = colorVal - } - return NewFragmentFormatter(color), nil -} - -func init() { - registry.RegisterFragmentFormatter(Name, Constructor) -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/html/fragment_formatter_html_test.go b/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/html/fragment_formatter_html_test.go deleted file mode 100644 index c5914f6f..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/fragment_formatters/html/fragment_formatter_html_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package html - -import ( - "testing" - - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/highlight" -) - -func TestHTMLFragmentFormatter1(t *testing.T) { - tests := []struct { - fragment *highlight.Fragment - tlm search.TermLocationMap - output string - }{ - { - fragment: &highlight.Fragment{ - Orig: []byte("the quick brown fox"), - Start: 0, - End: 19, - }, - tlm: search.TermLocationMap{ - "quick": search.Locations{ - &search.Location{ - Pos: 2, - Start: 4, - End: 9, - }, - }, - }, - output: "the quick brown fox", - }, - } - - emHTMLFormatter := NewFragmentFormatter("", "") - for _, test := range tests { - otl := highlight.OrderTermLocations(test.tlm) - result := emHTMLFormatter.Format(test.fragment, otl) - if result != test.output { - t.Errorf("expected `%s`, got `%s`", test.output, result) - } - } -} - -func TestHTMLFragmentFormatter2(t *testing.T) { - tests := []struct { - fragment *highlight.Fragment - tlm search.TermLocationMap - output string - }{ - { - fragment: &highlight.Fragment{ - Orig: []byte("the quick brown fox"), - Start: 0, - End: 19, - }, - tlm: search.TermLocationMap{ - "quick": search.Locations{ - &search.Location{ - Pos: 2, - Start: 4, - End: 9, - }, - }, - }, - output: "the quick brown fox", - }, - } - - emHTMLFormatter := NewFragmentFormatter("", "") - for _, test := range tests { - otl := highlight.OrderTermLocations(test.tlm) - result := emHTMLFormatter.Format(test.fragment, otl) - if result != test.output { - t.Errorf("expected `%s`, got `%s`", test.output, result) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenters/simple/fragmenter_simple.go b/vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go similarity index 82% rename from vendor/github.com/blevesearch/bleve/search/highlight/fragmenters/simple/fragmenter_simple.go rename to vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go index 55b33afe..6f6ecedf 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenters/simple/fragmenter_simple.go +++ b/vendor/github.com/blevesearch/bleve/search/highlight/fragmenter/simple/simple.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package simple @@ -31,8 +36,7 @@ func NewFragmenter(fragmentSize int) *Fragmenter { } func (s *Fragmenter) Fragment(orig []byte, ot highlight.TermLocations) []*highlight.Fragment { - rv := make([]*highlight.Fragment, 0) - + var rv []*highlight.Fragment maxbegin := 0 OUTER: for currTermIndex, termLocation := range ot { @@ -47,11 +51,11 @@ OUTER: continue OUTER // bail } end += size - used += 1 + used++ } // if we still have more characters available to us - // push back towards begining + // push back towards beginning // without cross maxbegin for start > 0 && used < s.fragmentSize { r, size := utf8.DecodeLastRune(orig[0:start]) @@ -60,7 +64,7 @@ OUTER: } if start-size >= maxbegin { start -= size - used += 1 + used++ } else { break } diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenters/simple/fragmenter_simple_test.go b/vendor/github.com/blevesearch/bleve/search/highlight/fragmenters/simple/fragmenter_simple_test.go deleted file mode 100644 index 03c20417..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/fragmenters/simple/fragmenter_simple_test.go +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package simple - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/search/highlight" -) - -func TestSimpleFragmenter(t *testing.T) { - - tests := []struct { - orig []byte - fragments []*highlight.Fragment - ot highlight.TermLocations - size int - }{ - { - orig: []byte("this is a test"), - fragments: []*highlight.Fragment{ - &highlight.Fragment{ - Orig: []byte("this is a test"), - Start: 0, - End: 14, - }, - }, - ot: highlight.TermLocations{ - &highlight.TermLocation{ - Term: "test", - Pos: 4, - Start: 10, - End: 14, - }, - }, - size: 100, - }, - { - orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"), - fragments: []*highlight.Fragment{ - &highlight.Fragment{ - Orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"), - Start: 0, - End: 100, - }, - }, - ot: highlight.TermLocations{ - &highlight.TermLocation{ - Term: "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", - Pos: 1, - Start: 0, - End: 100, - }, - }, - size: 100, - }, - { - orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - fragments: []*highlight.Fragment{ - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 0, - End: 100, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 10, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 20, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 30, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 40, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 50, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 60, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 70, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 80, - End: 101, - }, - &highlight.Fragment{ - Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), - Start: 90, - End: 101, - }, - }, - ot: highlight.TermLocations{ - &highlight.TermLocation{ - Term: "0123456789", - Pos: 1, - Start: 0, - End: 10, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 2, - Start: 10, - End: 20, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 3, - Start: 20, - End: 30, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 4, - Start: 30, - End: 40, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 5, - Start: 40, - End: 50, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 6, - Start: 50, - End: 60, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 7, - Start: 60, - End: 70, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 8, - Start: 70, - End: 80, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 9, - Start: 80, - End: 90, - }, - &highlight.TermLocation{ - Term: "0123456789", - Pos: 10, - Start: 90, - End: 100, - }, - }, - size: 100, - }, - { - orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"), - fragments: []*highlight.Fragment{ - &highlight.Fragment{ - Orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"), - Start: 0, - End: 411, - }, - }, - ot: highlight.TermLocations{ - &highlight.TermLocation{ - Term: "पानी", - Pos: 1, - Start: 2, - End: 14, - }, - }, - size: 200, - }, - { - orig: []byte("交换机"), - fragments: []*highlight.Fragment{ - &highlight.Fragment{ - Orig: []byte("交换机"), - Start: 0, - End: 9, - }, - &highlight.Fragment{ - Orig: []byte("交换机"), - Start: 3, - End: 9, - }, - }, - ot: highlight.TermLocations{ - &highlight.TermLocation{ - Term: "交换", - Pos: 1, - Start: 0, - End: 6, - }, - &highlight.TermLocation{ - Term: "换机", - Pos: 2, - Start: 3, - End: 9, - }, - }, - size: 200, - }, - } - - for _, test := range tests { - fragmenter := NewFragmenter(test.size) - fragments := fragmenter.Fragment(test.orig, test.ot) - if !reflect.DeepEqual(fragments, test.fragments) { - t.Errorf("expected %#v, got %#v", test.fragments, fragments) - for _, fragment := range fragments { - t.Logf("frag: %s", fragment.Orig[fragment.Start:fragment.End]) - t.Logf("frag: %d - %d", fragment.Start, fragment.End) - } - } - } -} - -func TestSimpleFragmenterWithSize(t *testing.T) { - - tests := []struct { - orig []byte - fragments []*highlight.Fragment - ot highlight.TermLocations - }{ - { - orig: []byte("this is a test"), - fragments: []*highlight.Fragment{ - &highlight.Fragment{ - Orig: []byte("this is a test"), - Start: 0, - End: 5, - }, - &highlight.Fragment{ - Orig: []byte("this is a test"), - Start: 9, - End: 14, - }, - }, - ot: highlight.TermLocations{ - &highlight.TermLocation{ - Term: "this", - Pos: 1, - Start: 0, - End: 5, - }, - &highlight.TermLocation{ - Term: "test", - Pos: 4, - Start: 10, - End: 14, - }, - }, - }, - } - - fragmenter := NewFragmenter(5) - for _, test := range tests { - fragments := fragmenter.Fragment(test.orig, test.ot) - if !reflect.DeepEqual(fragments, test.fragments) { - t.Errorf("expected %#v, got %#v", test.fragments, fragments) - for _, fragment := range fragments { - t.Logf("frag: %#v", fragment) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go index 2d178b3e..8077985d 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go +++ b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package highlight @@ -15,11 +20,12 @@ import ( ) type Fragment struct { - Orig []byte - Start int - End int - Score float64 - Index int // used by heap + Orig []byte + ArrayPositions []uint64 + Start int + End int + Score float64 + Index int // used by heap } func (f *Fragment) Overlaps(other *Fragment) bool { diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go new file mode 100644 index 00000000..928589c4 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/html/html.go @@ -0,0 +1,50 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package html + +import ( + "fmt" + + "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/search/highlight" + htmlFormatter "github.com/blevesearch/bleve/search/highlight/format/html" + simpleFragmenter "github.com/blevesearch/bleve/search/highlight/fragmenter/simple" + simpleHighlighter "github.com/blevesearch/bleve/search/highlight/highlighter/simple" +) + +const Name = "html" + +func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) { + + fragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragmenter: %v", err) + } + + formatter, err := cache.FragmentFormatterNamed(htmlFormatter.Name) + if err != nil { + return nil, fmt.Errorf("error building fragment formatter: %v", err) + } + + return simpleHighlighter.NewHighlighter( + fragmenter, + formatter, + simpleHighlighter.DefaultSeparator), + nil +} + +func init() { + registry.RegisterHighlighter(Name, Constructor) +} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/fragment_scorer_simple.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go similarity index 53% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/fragment_scorer_simple.go rename to vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go index 383beacc..3ec4c3d2 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/fragment_scorer_simple.go +++ b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/fragment_scorer_simple.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package simple @@ -32,7 +37,7 @@ func (s *FragmentScorer) Score(f *highlight.Fragment) { OUTER: for _, locations := range s.tlm { for _, location := range locations { - if int(location.Start) >= f.Start && int(location.End) <= f.End { + if location.ArrayPositions.Equals(f.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End { score += 1.0 // once we find a term in the fragment // don't care about additional matches diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/highlighter_simple.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go similarity index 82% rename from vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/highlighter_simple.go rename to vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go index e297fd4a..4849516b 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/highlighter_simple.go +++ b/vendor/github.com/blevesearch/bleve/search/highlight/highlighter/simple/highlighter_simple.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package simple @@ -20,7 +25,7 @@ import ( ) const Name = "simple" -const defaultSeparator = "…" +const DefaultSeparator = "…" type Highlighter struct { fragmenter highlight.Fragmenter @@ -80,9 +85,17 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume if f.Name() == field { _, ok := f.(*document.TextField) if ok { + termLocationsSameArrayPosition := make(highlight.TermLocations, 0) + for _, otl := range orderedTermLocations { + if otl.ArrayPositions.Equals(f.ArrayPositions()) { + termLocationsSameArrayPosition = append(termLocationsSameArrayPosition, otl) + } + } + fieldData := f.Value() - fragments := s.fragmenter.Fragment(fieldData, orderedTermLocations) + fragments := s.fragmenter.Fragment(fieldData, termLocationsSameArrayPosition) for _, fragment := range fragments { + fragment.ArrayPositions = f.ArrayPositions() scorer.Score(fragment) heap.Push(&fq, fragment) } @@ -91,7 +104,7 @@ func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc *docume } // now find the N best non-overlapping fragments - bestFragments := make([]*highlight.Fragment, 0) + var bestFragments []*highlight.Fragment if len(fq) > 0 { candidate := heap.Pop(&fq) OUTER: @@ -176,7 +189,7 @@ func (fq *FragmentQueue) Pop() interface{} { } func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) { - separator := defaultSeparator + separator := DefaultSeparator separatorVal, ok := config["separator"].(string) if ok { separator = separatorVal diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/fragment_scorer_simple_test.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/fragment_scorer_simple_test.go deleted file mode 100644 index 90374940..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/fragment_scorer_simple_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package simple - -import ( - "testing" - - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/highlight" -) - -func TestSimpleFragmentScorer(t *testing.T) { - - tests := []struct { - fragment *highlight.Fragment - tlm search.TermLocationMap - score float64 - }{ - { - fragment: &highlight.Fragment{ - Orig: []byte("cat in the hat"), - Start: 0, - End: 14, - }, - tlm: search.TermLocationMap{ - "cat": search.Locations{ - &search.Location{ - Pos: 0, - Start: 0, - End: 3, - }, - }, - }, - score: 1, - }, - { - fragment: &highlight.Fragment{ - Orig: []byte("cat in the hat"), - Start: 0, - End: 14, - }, - tlm: search.TermLocationMap{ - "cat": search.Locations{ - &search.Location{ - Pos: 1, - Start: 0, - End: 3, - }, - }, - "hat": search.Locations{ - &search.Location{ - Pos: 4, - Start: 11, - End: 14, - }, - }, - }, - score: 2, - }, - } - - for _, test := range tests { - scorer := NewFragmentScorer(test.tlm) - scorer.Score(test.fragment) - if test.fragment.Score != test.score { - t.Errorf("expected score %f, got %f", test.score, test.fragment.Score) - } - } - -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/highlighter_simple_test.go b/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/highlighter_simple_test.go deleted file mode 100644 index 8bc30342..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/highlighters/simple/highlighter_simple_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package simple - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/highlight/fragment_formatters/ansi" - sfrag "github.com/blevesearch/bleve/search/highlight/fragmenters/simple" -) - -const ( - reset = "\x1b[0m" - DefaultAnsiHighlight = "\x1b[43m" -) - -func TestSimpleHighlighter(t *testing.T) { - fragmenter := sfrag.NewFragmenter(100) - formatter := ansi.NewFragmentFormatter(ansi.DefaultAnsiHighlight) - highlighter := NewHighlighter(fragmenter, formatter, defaultSeparator) - - docMatch := search.DocumentMatch{ - ID: "a", - Score: 1.0, - Locations: search.FieldTermLocationMap{ - "desc": search.TermLocationMap{ - "quick": search.Locations{ - &search.Location{ - Pos: 2, - Start: 4, - End: 9, - }, - }, - "fox": search.Locations{ - &search.Location{ - Pos: 4, - Start: 16, - End: 19, - }, - }, - }, - }, - } - - expectedFragment := "the " + DefaultAnsiHighlight + "quick" + reset + " brown " + DefaultAnsiHighlight + "fox" + reset + " jumps over the lazy dog" - doc := document.NewDocument("a").AddField(document.NewTextField("desc", []uint64{}, []byte("the quick brown fox jumps over the lazy dog"))) - - fragment := highlighter.BestFragmentInField(&docMatch, doc, "desc") - if fragment != expectedFragment { - t.Errorf("expected `%s`, got `%s`", expectedFragment, fragment) - } -} - -func TestSimpleHighlighterLonger(t *testing.T) { - - fieldBytes := []byte(`Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris sed semper nulla, sed pellentesque urna. Suspendisse potenti. Aliquam dignissim pulvinar erat vel ullamcorper. Nullam sed diam at dolor dapibus varius. Vestibulum at semper nunc. Integer ullamcorper enim ut nisi condimentum lacinia. Nulla ipsum ipsum, dictum in dapibus non, bibendum eget neque. Vestibulum malesuada erat quis malesuada dictum. Mauris luctus viverra lorem, nec hendrerit lacus lacinia ut. Donec suscipit sit amet nisi et dictum. Maecenas ultrices mollis diam, vel commodo libero lobortis nec. Nunc non dignissim dolor. Nulla non tempus risus, eget porttitor lectus. Suspendisse vitae gravida magna, a sagittis urna. Curabitur nec dui volutpat, hendrerit nisi non, adipiscing erat. Maecenas aliquet sem sit amet nibh ultrices accumsan. - -Mauris lobortis sem sed blandit bibendum. In scelerisque eros sed metus aliquet convallis ac eget metus. Donec eget feugiat sem. Quisque venenatis, augue et blandit vulputate, velit odio viverra dolor, eu iaculis eros urna ut nunc. Duis faucibus mattis enim ut ultricies. Donec scelerisque volutpat elit, vel varius ante porttitor vel. Duis neque nulla, ultrices vel est id, molestie semper odio. Maecenas condimentum felis vitae nibh venenatis, ut feugiat risus vehicula. Suspendisse non sapien neque. Etiam et lorem consequat lorem aliquam ullamcorper. Pellentesque id vestibulum neque, at aliquam turpis. Aenean ultrices nec erat sit amet aliquam. Morbi eu sem in augue cursus ullamcorper a sed dolor. Integer et lobortis nulla, sit amet laoreet elit. In elementum, nibh nec volutpat pretium, lectus est pulvinar arcu, vehicula lobortis tellus sem id mauris. Maecenas ac blandit purus, sit amet scelerisque magna. - -In hac habitasse platea dictumst. In lacinia elit non risus venenatis viverra. Nulla vestibulum laoreet turpis ac accumsan. Vivamus eros felis, rhoncus vel interdum bibendum, imperdiet nec diam. Etiam sed eros sed orci pellentesque sagittis. Praesent a fermentum leo. Vivamus ipsum risus, faucibus a dignissim ut, ullamcorper nec risus. Etiam quis adipiscing velit. Nam ac cursus arcu. Sed bibendum lectus quis massa dapibus dapibus. Vestibulum fermentum eros vitae hendrerit condimentum. - -Fusce viverra eleifend iaculis. Maecenas tempor dictum cursus. Mauris faucibus, tortor in bibendum ornare, nibh lorem sollicitudin est, sed consectetur nulla dui imperdiet urna. Fusce aliquet odio fermentum massa mollis, id feugiat lacus egestas. Integer et eleifend metus. Duis neque tellus, vulputate nec dui eu, euismod sodales orci. Vivamus turpis erat, consectetur et pulvinar nec, ornare a quam. Maecenas fermentum, ligula vitae consectetur lobortis, mi lacus fermentum ante, ut semper lacus lectus porta orci. Nulla vehicula sodales eros, in iaculis ante laoreet at. Sed venenatis interdum metus, egestas scelerisque orci laoreet ut. Donec fermentum enim eget nibh blandit laoreet. Proin lacinia adipiscing lorem vel ornare. Donec ullamcorper massa elementum urna varius viverra. Proin pharetra, erat at feugiat rhoncus, velit eros condimentum mi, ac mattis sapien dolor non elit. Aenean viverra purus id tincidunt vulputate. - -Etiam vel augue vel nisl commodo suscipit et ac nisl. Quisque eros diam, porttitor et aliquet sed, vulputate in odio. Aenean feugiat est quis neque vehicula, eget vulputate nunc tempor. Donec quis nulla ut quam feugiat consectetur ut et justo. Nulla congue, metus auctor facilisis scelerisque, nunc risus vulputate urna, in blandit urna nibh et neque. Etiam quis tortor ut nulla dignissim dictum non sed ligula. Vivamus accumsan ligula eget ipsum ultrices, a tincidunt urna blandit. In hac habitasse platea dictumst.`) - - doc := document.NewDocument("a").AddField(document.NewTextField("full", []uint64{}, fieldBytes)) - docMatch := search.DocumentMatch{ - ID: "a", - Score: 1.0, - Locations: search.FieldTermLocationMap{ - "full": search.TermLocationMap{ - "metus": search.Locations{ - &search.Location{ - Pos: 0, - Start: 883, - End: 888, - }, - &search.Location{ - Pos: 0, - Start: 915, - End: 920, - }, - &search.Location{ - Pos: 0, - Start: 2492, - End: 2497, - }, - &search.Location{ - Pos: 0, - Start: 2822, - End: 2827, - }, - &search.Location{ - Pos: 0, - Start: 3417, - End: 3422, - }, - }, - "interdum": search.Locations{ - &search.Location{ - Pos: 0, - Start: 1891, - End: 1899, - }, - &search.Location{ - Pos: 0, - Start: 2813, - End: 2821, - }, - }, - "venenatis": search.Locations{ - &search.Location{ - Pos: 0, - Start: 954, - End: 963, - }, - &search.Location{ - Pos: 0, - Start: 1252, - End: 1261, - }, - &search.Location{ - Pos: 0, - Start: 1795, - End: 1804, - }, - &search.Location{ - Pos: 0, - Start: 2803, - End: 2812, - }, - }, - }, - }, - } - - expectedFragments := []string{ - "…eros, in iaculis ante laoreet at. Sed " + DefaultAnsiHighlight + "venenatis" + reset + " " + DefaultAnsiHighlight + "interdum" + reset + " " + DefaultAnsiHighlight + "metus" + reset + ", egestas scelerisque orci laoreet ut.…", - "… eros sed " + DefaultAnsiHighlight + "metus" + reset + " aliquet convallis ac eget " + DefaultAnsiHighlight + "metus" + reset + ". Donec eget feugiat sem. Quisque " + DefaultAnsiHighlight + "venenatis" + reset + ", augue et…", - "… odio. Maecenas condimentum felis vitae nibh " + DefaultAnsiHighlight + "venenatis" + reset + ", ut feugiat risus vehicula. Suspendisse non s…", - "… id feugiat lacus egestas. Integer et eleifend " + DefaultAnsiHighlight + "metus" + reset + ". Duis neque tellus, vulputate nec dui eu, euism…", - "… accumsan. Vivamus eros felis, rhoncus vel " + DefaultAnsiHighlight + "interdum" + reset + " bibendum, imperdiet nec diam. Etiam sed eros sed…", - } - - fragmenter := sfrag.NewFragmenter(100) - formatter := ansi.NewFragmentFormatter(ansi.DefaultAnsiHighlight) - highlighter := NewHighlighter(fragmenter, formatter, defaultSeparator) - fragments := highlighter.BestFragmentsInField(&docMatch, doc, "full", 5) - - if !reflect.DeepEqual(fragments, expectedFragments) { - t.Errorf("expected %#v, got %#v", expectedFragments, fragments) - } - -} diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go b/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go index 4a9817c9..6d2cb133 100644 --- a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go +++ b/vendor/github.com/blevesearch/bleve/search/highlight/term_locations.go @@ -1,41 +1,75 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package highlight import ( + "reflect" "sort" "github.com/blevesearch/bleve/search" ) type TermLocation struct { - Term string - Pos int - Start int - End int + Term string + ArrayPositions search.ArrayPositions + Pos int + Start int + End int } func (tl *TermLocation) Overlaps(other *TermLocation) bool { - if other.Start >= tl.Start && other.Start < tl.End { - return true - } else if tl.Start >= other.Start && tl.Start < other.End { - return true + if reflect.DeepEqual(tl.ArrayPositions, other.ArrayPositions) { + if other.Start >= tl.Start && other.Start < tl.End { + return true + } else if tl.Start >= other.Start && tl.Start < other.End { + return true + } } return false } type TermLocations []*TermLocation -func (t TermLocations) Len() int { return len(t) } -func (t TermLocations) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t TermLocations) Less(i, j int) bool { return t[i].Start < t[j].Start } +func (t TermLocations) Len() int { return len(t) } +func (t TermLocations) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t TermLocations) Less(i, j int) bool { + + shortestArrayPositions := len(t[i].ArrayPositions) + if len(t[j].ArrayPositions) < shortestArrayPositions { + shortestArrayPositions = len(t[j].ArrayPositions) + } + + // compare all the common array positions + for api := 0; api < shortestArrayPositions; api++ { + if t[i].ArrayPositions[api] < t[j].ArrayPositions[api] { + return true + } + if t[i].ArrayPositions[api] > t[j].ArrayPositions[api] { + return false + } + } + // all the common array positions are the same + if len(t[i].ArrayPositions) < len(t[j].ArrayPositions) { + return true // j array positions, longer so greater + } else if len(t[i].ArrayPositions) > len(t[j].ArrayPositions) { + return false // j array positions, shorter so less + } + + // array positions the same, compare starts + return t[i].Start < t[j].Start +} func (t TermLocations) MergeOverlapping() { var lastTl *TermLocation @@ -57,10 +91,11 @@ func OrderTermLocations(tlm search.TermLocationMap) TermLocations { for term, locations := range tlm { for _, location := range locations { tl := TermLocation{ - Term: term, - Pos: int(location.Pos), - Start: int(location.Start), - End: int(location.End), + Term: term, + ArrayPositions: location.ArrayPositions, + Pos: int(location.Pos), + Start: int(location.Start), + End: int(location.End), } rv = append(rv, &tl) } diff --git a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations_test.go b/vendor/github.com/blevesearch/bleve/search/highlight/term_locations_test.go deleted file mode 100644 index edaf315c..00000000 --- a/vendor/github.com/blevesearch/bleve/search/highlight/term_locations_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package highlight - -import ( - "reflect" - "testing" -) - -func TestTermLocationOverlaps(t *testing.T) { - - tests := []struct { - left *TermLocation - right *TermLocation - expected bool - }{ - { - left: &TermLocation{ - Start: 0, - End: 5, - }, - right: &TermLocation{ - Start: 3, - End: 7, - }, - expected: true, - }, - { - left: &TermLocation{ - Start: 0, - End: 5, - }, - right: &TermLocation{ - Start: 5, - End: 7, - }, - expected: false, - }, - { - left: &TermLocation{ - Start: 0, - End: 5, - }, - right: &TermLocation{ - Start: 7, - End: 11, - }, - expected: false, - }, - } - - for _, test := range tests { - actual := test.left.Overlaps(test.right) - if actual != test.expected { - t.Errorf("expected %t got %t for %#v", test.expected, actual, test) - } - } -} - -func TestTermLocationsMergeOverlapping(t *testing.T) { - - tests := []struct { - input TermLocations - output TermLocations - }{ - { - input: TermLocations{}, - output: TermLocations{}, - }, - { - input: TermLocations{ - &TermLocation{ - Start: 0, - End: 5, - }, - &TermLocation{ - Start: 7, - End: 11, - }, - }, - output: TermLocations{ - &TermLocation{ - Start: 0, - End: 5, - }, - &TermLocation{ - Start: 7, - End: 11, - }, - }, - }, - { - input: TermLocations{ - &TermLocation{ - Start: 0, - End: 5, - }, - &TermLocation{ - Start: 4, - End: 11, - }, - }, - output: TermLocations{ - &TermLocation{ - Start: 0, - End: 11, - }, - nil, - }, - }, - { - input: TermLocations{ - &TermLocation{ - Start: 0, - End: 5, - }, - &TermLocation{ - Start: 4, - End: 11, - }, - &TermLocation{ - Start: 9, - End: 13, - }, - }, - output: TermLocations{ - &TermLocation{ - Start: 0, - End: 13, - }, - nil, - nil, - }, - }, - { - input: TermLocations{ - &TermLocation{ - Start: 0, - End: 5, - }, - &TermLocation{ - Start: 4, - End: 11, - }, - &TermLocation{ - Start: 9, - End: 13, - }, - &TermLocation{ - Start: 15, - End: 21, - }, - }, - output: TermLocations{ - &TermLocation{ - Start: 0, - End: 13, - }, - nil, - nil, - &TermLocation{ - Start: 15, - End: 21, - }, - }, - }, - } - - for _, test := range tests { - test.input.MergeOverlapping() - if !reflect.DeepEqual(test.input, test.output) { - t.Errorf("expected: %#v got %#v", test.output, test.input) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/levenshtein.go b/vendor/github.com/blevesearch/bleve/search/levenshtein.go index 7a647096..ec033143 100644 --- a/vendor/github.com/blevesearch/bleve/search/levenshtein.go +++ b/vendor/github.com/blevesearch/bleve/search/levenshtein.go @@ -1,12 +1,26 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package search import ( "math" ) -func LevenshteinDistance(a, b *string) int { - la := len(*a) - lb := len(*b) +func LevenshteinDistance(a, b string) int { + la := len(a) + lb := len(b) d := make([]int, la+1) var lastdiag, olddiag, temp int @@ -22,7 +36,7 @@ func LevenshteinDistance(a, b *string) int { if (d[j-1] + 1) < min { min = d[j-1] + 1 } - if (*a)[j-1] == (*b)[i-1] { + if a[j-1] == b[i-1] { temp = 0 } else { temp = 1 @@ -37,14 +51,14 @@ func LevenshteinDistance(a, b *string) int { return d[la] } -// levenshteinDistanceMax same as levenshteinDistance but +// LevenshteinDistanceMax same as LevenshteinDistance but // attempts to bail early once we know the distance // will be greater than max // in which case the first return val will be the max // and the second will be true, indicating max was exceeded -func LevenshteinDistanceMax(a, b *string, max int) (int, bool) { - la := len(*a) - lb := len(*b) +func LevenshteinDistanceMax(a, b string, max int) (int, bool) { + la := len(a) + lb := len(b) ld := int(math.Abs(float64(la - lb))) if ld > max { @@ -67,7 +81,7 @@ func LevenshteinDistanceMax(a, b *string, max int) (int, bool) { if (d[j-1] + 1) < min { min = d[j-1] + 1 } - if (*a)[j-1] == (*b)[i-1] { + if a[j-1] == b[i-1] { temp = 0 } else { temp = 1 @@ -82,7 +96,7 @@ func LevenshteinDistanceMax(a, b *string, max int) (int, bool) { lastdiag = olddiag } - // after each row if rowmin isnt less than max stop + // after each row if rowmin isn't less than max stop if rowmin > max { return max, true } diff --git a/vendor/github.com/blevesearch/bleve/search/levenshtein_test.go b/vendor/github.com/blevesearch/bleve/search/levenshtein_test.go deleted file mode 100644 index 7547b956..00000000 --- a/vendor/github.com/blevesearch/bleve/search/levenshtein_test.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package search - -import ( - "testing" -) - -func TestLevenshteinDistance(t *testing.T) { - - tests := []struct { - a string - b string - dist int - }{ - { - "water", - "atec", - 2, - }, - { - "water", - "aphex", - 4, - }, - } - - for _, test := range tests { - actual := LevenshteinDistance(&test.a, &test.b) - if actual != test.dist { - t.Errorf("expected %d, got %d for %s and %s", test.dist, actual, test.a, test.b) - } - } -} - -func TestLevenshteinDistanceMax(t *testing.T) { - - tests := []struct { - a string - b string - max int - dist int - exceeded bool - }{ - { - a: "water", - b: "atec", - max: 1, - dist: 1, - exceeded: true, - }, - { - a: "water", - b: "christmas", - max: 3, - dist: 3, - exceeded: true, - }, - { - a: "water", - b: "water", - max: 1, - dist: 0, - exceeded: false, - }, - } - - for _, test := range tests { - actual, exceeded := LevenshteinDistanceMax(&test.a, &test.b, test.max) - if actual != test.dist || exceeded != test.exceeded { - t.Errorf("expected %d %t, got %d %t for %s and %s", test.dist, test.exceeded, actual, exceeded, test.a, test.b) - } - } -} - -// 5 terms that are less than 2 -// 5 terms that are more than 2 -var benchmarkTerms = []string{ - "watex", - "aters", - "wayer", - "wbter", - "yater", - "christmas", - "waterwaterwater", - "watcatdogfish", - "q", - "couchbase", -} - -func BenchmarkLevenshteinDistance(b *testing.B) { - a := "water" - for i := 0; i < b.N; i++ { - for _, t := range benchmarkTerms { - LevenshteinDistance(&a, &t) - } - } -} - -func BenchmarkLevenshteinDistanceMax(b *testing.B) { - a := "water" - for i := 0; i < b.N; i++ { - for _, t := range benchmarkTerms { - LevenshteinDistanceMax(&a, &t, 2) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/pool.go b/vendor/github.com/blevesearch/bleve/search/pool.go new file mode 100644 index 00000000..b9b52a61 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/pool.go @@ -0,0 +1,80 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +// DocumentMatchPoolTooSmall is a callback function that can be executed +// when the DocumentMatchPool does not have sufficient capacity +// By default we just perform just-in-time allocation, but you could log +// a message, or panic, etc. +type DocumentMatchPoolTooSmall func(p *DocumentMatchPool) *DocumentMatch + +// DocumentMatchPool manages use/re-use of DocumentMatch instances +// it pre-allocates space from a single large block with the expected +// number of instances. It is not thread-safe as currently all +// aspects of search take place in a single goroutine. +type DocumentMatchPool struct { + avail DocumentMatchCollection + TooSmall DocumentMatchPoolTooSmall +} + +func defaultDocumentMatchPoolTooSmall(p *DocumentMatchPool) *DocumentMatch { + return &DocumentMatch{} +} + +// NewDocumentMatchPool will build a DocumentMatchPool with memory +// pre-allocated to accommodate the requested number of DocumentMatch +// instances +func NewDocumentMatchPool(size, sortsize int) *DocumentMatchPool { + avail := make(DocumentMatchCollection, size) + // pre-allocate the expected number of instances + startBlock := make([]DocumentMatch, size) + startSorts := make([]string, size*sortsize) + // make these initial instances available + i, j := 0, 0 + for i < size { + avail[i] = &startBlock[i] + avail[i].Sort = startSorts[j:j] + i += 1 + j += sortsize + } + return &DocumentMatchPool{ + avail: avail, + TooSmall: defaultDocumentMatchPoolTooSmall, + } +} + +// Get returns an available DocumentMatch from the pool +// if the pool was not allocated with sufficient size, an allocation will +// occur to satisfy this request. As a side-effect this will grow the size +// of the pool. +func (p *DocumentMatchPool) Get() *DocumentMatch { + var rv *DocumentMatch + if len(p.avail) > 0 { + rv, p.avail = p.avail[len(p.avail)-1], p.avail[:len(p.avail)-1] + } else { + rv = p.TooSmall(p) + } + return rv +} + +// Put returns a DocumentMatch to the pool +func (p *DocumentMatchPool) Put(d *DocumentMatch) { + if d == nil { + return + } + // reset DocumentMatch before returning it to available pool + d.Reset() + p.avail = append(p.avail, d) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/bool_field.go b/vendor/github.com/blevesearch/bleve/search/query/bool_field.go new file mode 100644 index 00000000..b7b5a3d3 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/bool_field.go @@ -0,0 +1,64 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type BoolFieldQuery struct { + Bool bool `json:"bool"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewBoolFieldQuery creates a new Query for boolean fields +func NewBoolFieldQuery(val bool) *BoolFieldQuery { + return &BoolFieldQuery{ + Bool: val, + } +} + +func (q *BoolFieldQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *BoolFieldQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *BoolFieldQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *BoolFieldQuery) Field() string { + return q.FieldVal +} + +func (q *BoolFieldQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + term := "F" + if q.Bool { + term = "T" + } + return searcher.NewTermSearcher(i, term, field, q.BoostVal.Value(), options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/boolean.go b/vendor/github.com/blevesearch/bleve/search/query/boolean.go new file mode 100644 index 00000000..3cfa1d99 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/boolean.go @@ -0,0 +1,248 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type BooleanQuery struct { + Must Query `json:"must,omitempty"` + Should Query `json:"should,omitempty"` + MustNot Query `json:"must_not,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + queryStringMode bool +} + +// NewBooleanQuery creates a compound Query composed +// of several other Query objects. +// Result documents must satisfy ALL of the +// must Queries. +// Result documents must satisfy NONE of the must not +// Queries. +// Result documents that ALSO satisfy any of the should +// Queries will score higher. +func NewBooleanQuery(must []Query, should []Query, mustNot []Query) *BooleanQuery { + + rv := BooleanQuery{} + if len(must) > 0 { + rv.Must = NewConjunctionQuery(must) + } + if len(should) > 0 { + rv.Should = NewDisjunctionQuery(should) + } + if len(mustNot) > 0 { + rv.MustNot = NewDisjunctionQuery(mustNot) + } + + return &rv +} + +func NewBooleanQueryForQueryString(must []Query, should []Query, mustNot []Query) *BooleanQuery { + rv := NewBooleanQuery(nil, nil, nil) + rv.queryStringMode = true + rv.AddMust(must...) + rv.AddShould(should...) + rv.AddMustNot(mustNot...) + return rv +} + +// SetMinShould requires that at least minShould of the +// should Queries must be satisfied. +func (q *BooleanQuery) SetMinShould(minShould float64) { + q.Should.(*DisjunctionQuery).SetMin(minShould) +} + +func (q *BooleanQuery) AddMust(m ...Query) { + if q.Must == nil { + tmp := NewConjunctionQuery([]Query{}) + tmp.queryStringMode = q.queryStringMode + q.Must = tmp + } + for _, mq := range m { + q.Must.(*ConjunctionQuery).AddQuery(mq) + } +} + +func (q *BooleanQuery) AddShould(m ...Query) { + if q.Should == nil { + tmp := NewDisjunctionQuery([]Query{}) + tmp.queryStringMode = q.queryStringMode + q.Should = tmp + } + for _, mq := range m { + q.Should.(*DisjunctionQuery).AddQuery(mq) + } +} + +func (q *BooleanQuery) AddMustNot(m ...Query) { + if q.MustNot == nil { + tmp := NewDisjunctionQuery([]Query{}) + tmp.queryStringMode = q.queryStringMode + q.MustNot = tmp + } + for _, mq := range m { + q.MustNot.(*DisjunctionQuery).AddQuery(mq) + } +} + +func (q *BooleanQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *BooleanQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *BooleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + var err error + var mustNotSearcher search.Searcher + if q.MustNot != nil { + mustNotSearcher, err = q.MustNot.Searcher(i, m, options) + if err != nil { + return nil, err + } + // if must not is MatchNone, reset it to nil + if _, ok := mustNotSearcher.(*searcher.MatchNoneSearcher); ok { + mustNotSearcher = nil + } + } + + var mustSearcher search.Searcher + if q.Must != nil { + mustSearcher, err = q.Must.Searcher(i, m, options) + if err != nil { + return nil, err + } + // if must searcher is MatchNone, reset it to nil + if _, ok := mustSearcher.(*searcher.MatchNoneSearcher); ok { + mustSearcher = nil + } + } + + var shouldSearcher search.Searcher + if q.Should != nil { + shouldSearcher, err = q.Should.Searcher(i, m, options) + if err != nil { + return nil, err + } + // if should searcher is MatchNone, reset it to nil + if _, ok := shouldSearcher.(*searcher.MatchNoneSearcher); ok { + shouldSearcher = nil + } + } + + // if all 3 are nil, return MatchNone + if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher == nil { + return searcher.NewMatchNoneSearcher(i) + } + + // if only mustNotSearcher, start with MatchAll + if mustSearcher == nil && shouldSearcher == nil && mustNotSearcher != nil { + mustSearcher, err = searcher.NewMatchAllSearcher(i, 1.0, options) + if err != nil { + return nil, err + } + } + + // optimization, if only should searcher, just return it instead + if mustSearcher == nil && shouldSearcher != nil && mustNotSearcher == nil { + return shouldSearcher, nil + } + + return searcher.NewBooleanSearcher(i, mustSearcher, shouldSearcher, mustNotSearcher, options) +} + +func (q *BooleanQuery) Validate() error { + if qm, ok := q.Must.(ValidatableQuery); ok { + err := qm.Validate() + if err != nil { + return err + } + } + if qs, ok := q.Should.(ValidatableQuery); ok { + err := qs.Validate() + if err != nil { + return err + } + } + if qmn, ok := q.MustNot.(ValidatableQuery); ok { + err := qmn.Validate() + if err != nil { + return err + } + } + if q.Must == nil && q.Should == nil && q.MustNot == nil { + return fmt.Errorf("boolean query must contain at least one must or should or not must clause") + } + return nil +} + +func (q *BooleanQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Must json.RawMessage `json:"must,omitempty"` + Should json.RawMessage `json:"should,omitempty"` + MustNot json.RawMessage `json:"must_not,omitempty"` + Boost *Boost `json:"boost,omitempty"` + }{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + + if tmp.Must != nil { + q.Must, err = ParseQuery(tmp.Must) + if err != nil { + return err + } + _, isConjunctionQuery := q.Must.(*ConjunctionQuery) + if !isConjunctionQuery { + return fmt.Errorf("must clause must be conjunction") + } + } + + if tmp.Should != nil { + q.Should, err = ParseQuery(tmp.Should) + if err != nil { + return err + } + _, isDisjunctionQuery := q.Should.(*DisjunctionQuery) + if !isDisjunctionQuery { + return fmt.Errorf("should clause must be disjunction") + } + } + + if tmp.MustNot != nil { + q.MustNot, err = ParseQuery(tmp.MustNot) + if err != nil { + return err + } + _, isDisjunctionQuery := q.MustNot.(*DisjunctionQuery) + if !isDisjunctionQuery { + return fmt.Errorf("must not clause must be disjunction") + } + } + + q.BoostVal = tmp.Boost + + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/boost.go b/vendor/github.com/blevesearch/bleve/search/query/boost.go new file mode 100644 index 00000000..13659945 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/boost.go @@ -0,0 +1,33 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import "fmt" + +type Boost float64 + +func (b *Boost) Value() float64 { + if b == nil { + return 1.0 + } + return float64(*b) +} + +func (b *Boost) GoString() string { + if b == nil { + return "boost unspecified" + } + return fmt.Sprintf("%f", *b) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/conjunction.go b/vendor/github.com/blevesearch/bleve/search/query/conjunction.go new file mode 100644 index 00000000..39cc312d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/conjunction.go @@ -0,0 +1,110 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type ConjunctionQuery struct { + Conjuncts []Query `json:"conjuncts"` + BoostVal *Boost `json:"boost,omitempty"` + queryStringMode bool +} + +// NewConjunctionQuery creates a new compound Query. +// Result documents must satisfy all of the queries. +func NewConjunctionQuery(conjuncts []Query) *ConjunctionQuery { + return &ConjunctionQuery{ + Conjuncts: conjuncts, + } +} + +func (q *ConjunctionQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *ConjunctionQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *ConjunctionQuery) AddQuery(aq ...Query) { + for _, aaq := range aq { + q.Conjuncts = append(q.Conjuncts, aaq) + } +} + +func (q *ConjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + ss := make([]search.Searcher, 0, len(q.Conjuncts)) + for _, conjunct := range q.Conjuncts { + sr, err := conjunct.Searcher(i, m, options) + if err != nil { + for _, searcher := range ss { + if searcher != nil { + _ = searcher.Close() + } + } + return nil, err + } + if _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode { + // in query string mode, skip match none + continue + } + ss = append(ss, sr) + } + if len(ss) < 1 { + return searcher.NewMatchNoneSearcher(i) + } + return searcher.NewConjunctionSearcher(i, ss, options) +} + +func (q *ConjunctionQuery) Validate() error { + for _, q := range q.Conjuncts { + if q, ok := q.(ValidatableQuery); ok { + err := q.Validate() + if err != nil { + return err + } + } + } + return nil +} + +func (q *ConjunctionQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Conjuncts []json.RawMessage `json:"conjuncts"` + Boost *Boost `json:"boost,omitempty"` + }{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + q.Conjuncts = make([]Query, len(tmp.Conjuncts)) + for i, term := range tmp.Conjuncts { + query, err := ParseQuery(term) + if err != nil { + return err + } + q.Conjuncts[i] = query + } + q.BoostVal = tmp.Boost + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/date_range.go b/vendor/github.com/blevesearch/bleve/search/query/date_range.go new file mode 100644 index 00000000..ff67a7bb --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/date_range.go @@ -0,0 +1,164 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + "math" + "time" + + "github.com/blevesearch/bleve/analysis/datetime/optional" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/registry" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +// QueryDateTimeParser controls the default query date time parser +var QueryDateTimeParser = optional.Name + +// QueryDateTimeFormat controls the format when Marshaling to JSON +var QueryDateTimeFormat = time.RFC3339 + +var cache = registry.NewCache() + +type BleveQueryTime struct { + time.Time +} + +func queryTimeFromString(t string) (time.Time, error) { + dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser) + if err != nil { + return time.Time{}, err + } + rv, err := dateTimeParser.ParseDateTime(t) + if err != nil { + return time.Time{}, err + } + return rv, nil +} + +func (t *BleveQueryTime) MarshalJSON() ([]byte, error) { + tt := time.Time(t.Time) + return []byte("\"" + tt.Format(QueryDateTimeFormat) + "\""), nil +} + +func (t *BleveQueryTime) UnmarshalJSON(data []byte) error { + var timeString string + err := json.Unmarshal(data, &timeString) + if err != nil { + return err + } + dateTimeParser, err := cache.DateTimeParserNamed(QueryDateTimeParser) + if err != nil { + return err + } + t.Time, err = dateTimeParser.ParseDateTime(timeString) + if err != nil { + return err + } + return nil +} + +type DateRangeQuery struct { + Start BleveQueryTime `json:"start,omitempty"` + End BleveQueryTime `json:"end,omitempty"` + InclusiveStart *bool `json:"inclusive_start,omitempty"` + InclusiveEnd *bool `json:"inclusive_end,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewDateRangeQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// top-level config.QueryDateTimeParser +// Either, but not both endpoints can be nil. +func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { + return NewDateRangeInclusiveQuery(start, end, nil, nil) +} + +// NewDateRangeInclusiveQuery creates a new Query for ranges +// of date values. +// Date strings are parsed using the DateTimeParser configured in the +// top-level config.QueryDateTimeParser +// Either, but not both endpoints can be nil. +// startInclusive and endInclusive control inclusion of the endpoints. +func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery { + return &DateRangeQuery{ + Start: BleveQueryTime{start}, + End: BleveQueryTime{end}, + InclusiveStart: startInclusive, + InclusiveEnd: endInclusive, + } +} + +func (q *DateRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DateRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DateRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *DateRangeQuery) Field() string { + return q.FieldVal +} + +func (q *DateRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + min, max, err := q.parseEndpoints() + if err != nil { + return nil, err + } + + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + return searcher.NewNumericRangeSearcher(i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) +} + +func (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) { + min := math.Inf(-1) + max := math.Inf(1) + if !q.Start.IsZero() { + min = numeric.Int64ToFloat64(q.Start.UnixNano()) + } + if !q.End.IsZero() { + max = numeric.Int64ToFloat64(q.End.UnixNano()) + } + + return &min, &max, nil +} + +func (q *DateRangeQuery) Validate() error { + if q.Start.IsZero() && q.End.IsZero() { + return fmt.Errorf("must specify start or end") + } + _, _, err := q.parseEndpoints() + if err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/disjunction.go b/vendor/github.com/blevesearch/bleve/search/query/disjunction.go new file mode 100644 index 00000000..dacc3a75 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/disjunction.go @@ -0,0 +1,121 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type DisjunctionQuery struct { + Disjuncts []Query `json:"disjuncts"` + BoostVal *Boost `json:"boost,omitempty"` + Min float64 `json:"min"` + queryStringMode bool +} + +// NewDisjunctionQuery creates a new compound Query. +// Result documents satisfy at least one Query. +func NewDisjunctionQuery(disjuncts []Query) *DisjunctionQuery { + return &DisjunctionQuery{ + Disjuncts: disjuncts, + } +} + +func (q *DisjunctionQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DisjunctionQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DisjunctionQuery) AddQuery(aq ...Query) { + for _, aaq := range aq { + q.Disjuncts = append(q.Disjuncts, aaq) + } +} + +func (q *DisjunctionQuery) SetMin(m float64) { + q.Min = m +} + +func (q *DisjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + ss := make([]search.Searcher, 0, len(q.Disjuncts)) + for _, disjunct := range q.Disjuncts { + sr, err := disjunct.Searcher(i, m, options) + if err != nil { + for _, searcher := range ss { + if searcher != nil { + _ = searcher.Close() + } + } + return nil, err + } + if _, ok := sr.(*searcher.MatchNoneSearcher); ok && q.queryStringMode { + // in query string mode, skip match none + continue + } + ss = append(ss, sr) + } + if len(ss) < 1 { + return searcher.NewMatchNoneSearcher(i) + } + return searcher.NewDisjunctionSearcher(i, ss, q.Min, options) +} + +func (q *DisjunctionQuery) Validate() error { + if int(q.Min) > len(q.Disjuncts) { + return fmt.Errorf("disjunction query has fewer than the minimum number of clauses to satisfy") + } + for _, q := range q.Disjuncts { + if q, ok := q.(ValidatableQuery); ok { + err := q.Validate() + if err != nil { + return err + } + } + } + return nil +} + +func (q *DisjunctionQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Disjuncts []json.RawMessage `json:"disjuncts"` + Boost *Boost `json:"boost,omitempty"` + Min float64 `json:"min"` + }{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + q.Disjuncts = make([]Query, len(tmp.Disjuncts)) + for i, term := range tmp.Disjuncts { + query, err := ParseQuery(term) + if err != nil { + return err + } + q.Disjuncts[i] = query + } + q.BoostVal = tmp.Boost + q.Min = tmp.Min + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/docid.go b/vendor/github.com/blevesearch/bleve/search/query/docid.go new file mode 100644 index 00000000..3b865f93 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/docid.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type DocIDQuery struct { + IDs []string `json:"ids"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewDocIDQuery creates a new Query object returning indexed documents among +// the specified set. Combine it with ConjunctionQuery to restrict the scope of +// other queries output. +func NewDocIDQuery(ids []string) *DocIDQuery { + return &DocIDQuery{ + IDs: ids, + } +} + +func (q *DocIDQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *DocIDQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *DocIDQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewDocIDSearcher(i, q.IDs, q.BoostVal.Value(), options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/fuzzy.go b/vendor/github.com/blevesearch/bleve/search/query/fuzzy.go new file mode 100644 index 00000000..f18982d4 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/fuzzy.go @@ -0,0 +1,77 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type FuzzyQuery struct { + Term string `json:"term"` + Prefix int `json:"prefix_length"` + Fuzziness int `json:"fuzziness"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewFuzzyQuery creates a new Query which finds +// documents containing terms within a specific +// fuzziness of the specified term. +// The default fuzziness is 1. +// +// The current implementation uses Levenshtein edit +// distance as the fuzziness metric. +func NewFuzzyQuery(term string) *FuzzyQuery { + return &FuzzyQuery{ + Term: term, + Fuzziness: 1, + } +} + +func (q *FuzzyQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *FuzzyQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *FuzzyQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *FuzzyQuery) Field() string { + return q.FieldVal +} + +func (q *FuzzyQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *FuzzyQuery) SetPrefix(p int) { + q.Prefix = p +} + +func (q *FuzzyQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + return searcher.NewFuzzySearcher(i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go b/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go new file mode 100644 index 00000000..de6be4a5 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/geo_boundingbox.go @@ -0,0 +1,113 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/geo" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type GeoBoundingBoxQuery struct { + TopLeft []float64 `json:"top_left,omitempty"` + BottomRight []float64 `json:"bottom_right,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64) *GeoBoundingBoxQuery { + return &GeoBoundingBoxQuery{ + TopLeft: []float64{topLeftLon, topLeftLat}, + BottomRight: []float64{bottomRightLon, bottomRightLat}, + } +} + +func (q *GeoBoundingBoxQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoBoundingBoxQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoBoundingBoxQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoBoundingBoxQuery) Field() string { + return q.FieldVal +} + +func (q *GeoBoundingBoxQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + if q.BottomRight[0] < q.TopLeft[0] { + // cross date line, rewrite as two parts + + leftSearcher, err := searcher.NewGeoBoundingBoxSearcher(i, -180, q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) + if err != nil { + return nil, err + } + rightSearcher, err := searcher.NewGeoBoundingBoxSearcher(i, q.TopLeft[0], q.BottomRight[1], 180, q.TopLeft[1], field, q.BoostVal.Value(), options, true) + if err != nil { + _ = leftSearcher.Close() + return nil, err + } + + return searcher.NewDisjunctionSearcher(i, []search.Searcher{leftSearcher, rightSearcher}, 0, options) + } + + return searcher.NewGeoBoundingBoxSearcher(i, q.TopLeft[0], q.BottomRight[1], q.BottomRight[0], q.TopLeft[1], field, q.BoostVal.Value(), options, true) +} + +func (q *GeoBoundingBoxQuery) Validate() error { + return nil +} + +func (q *GeoBoundingBoxQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + TopLeft interface{} `json:"top_left,omitempty"` + BottomRight interface{} `json:"bottom_right,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + }{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + // now use our generic point parsing code from the geo package + lon, lat, found := geo.ExtractGeoPoint(tmp.TopLeft) + if !found { + return fmt.Errorf("geo location top_left not in a valid format") + } + q.TopLeft = []float64{lon, lat} + lon, lat, found = geo.ExtractGeoPoint(tmp.BottomRight) + if !found { + return fmt.Errorf("geo location bottom_right not in a valid format") + } + q.BottomRight = []float64{lon, lat} + q.FieldVal = tmp.FieldVal + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/geo_distance.go b/vendor/github.com/blevesearch/bleve/search/query/geo_distance.go new file mode 100644 index 00000000..ef3aa88c --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/geo_distance.go @@ -0,0 +1,100 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/geo" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type GeoDistanceQuery struct { + Location []float64 `json:"location,omitempty"` + Distance string `json:"distance,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +func NewGeoDistanceQuery(lon, lat float64, distance string) *GeoDistanceQuery { + return &GeoDistanceQuery{ + Location: []float64{lon, lat}, + Distance: distance, + } +} + +func (q *GeoDistanceQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *GeoDistanceQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *GeoDistanceQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *GeoDistanceQuery) Field() string { + return q.FieldVal +} + +func (q *GeoDistanceQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + dist, err := geo.ParseDistance(q.Distance) + if err != nil { + return nil, err + } + + return searcher.NewGeoPointDistanceSearcher(i, q.Location[0], q.Location[1], + dist, field, q.BoostVal.Value(), options) +} + +func (q *GeoDistanceQuery) Validate() error { + return nil +} + +func (q *GeoDistanceQuery) UnmarshalJSON(data []byte) error { + tmp := struct { + Location interface{} `json:"location,omitempty"` + Distance string `json:"distance,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + }{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + // now use our generic point parsing code from the geo package + lon, lat, found := geo.ExtractGeoPoint(tmp.Location) + if !found { + return fmt.Errorf("geo location not in a valid format") + } + q.Location = []float64{lon, lat} + q.Distance = tmp.Distance + q.FieldVal = tmp.FieldVal + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/match.go b/vendor/github.com/blevesearch/bleve/search/query/match.go new file mode 100644 index 00000000..36c9ee4a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/match.go @@ -0,0 +1,176 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" +) + +type MatchQuery struct { + Match string `json:"match"` + FieldVal string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + Prefix int `json:"prefix_length"` + Fuzziness int `json:"fuzziness"` + Operator MatchQueryOperator `json:"operator,omitempty"` +} + +type MatchQueryOperator int + +const ( + // Document must satisfy AT LEAST ONE of term searches. + MatchQueryOperatorOr = 0 + // Document must satisfy ALL of term searches. + MatchQueryOperatorAnd = 1 +) + +func (o MatchQueryOperator) MarshalJSON() ([]byte, error) { + switch o { + case MatchQueryOperatorOr: + return json.Marshal("or") + case MatchQueryOperatorAnd: + return json.Marshal("and") + default: + return nil, fmt.Errorf("cannot marshal match operator %d to JSON", o) + } +} + +func (o *MatchQueryOperator) UnmarshalJSON(data []byte) error { + var operatorString string + err := json.Unmarshal(data, &operatorString) + if err != nil { + return err + } + + switch operatorString { + case "or": + *o = MatchQueryOperatorOr + return nil + case "and": + *o = MatchQueryOperatorAnd + return nil + default: + return fmt.Errorf("cannot unmarshal match operator '%v' from JSON", o) + } +} + +// NewMatchQuery creates a Query for matching text. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to perform term searches. Result documents +// must satisfy at least one of these term searches. +func NewMatchQuery(match string) *MatchQuery { + return &MatchQuery{ + Match: match, + Operator: MatchQueryOperatorOr, + } +} + +func (q *MatchQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *MatchQuery) Field() string { + return q.FieldVal +} + +func (q *MatchQuery) SetFuzziness(f int) { + q.Fuzziness = f +} + +func (q *MatchQuery) SetPrefix(p int) { + q.Prefix = p +} + +func (q *MatchQuery) SetOperator(operator MatchQueryOperator) { + q.Operator = operator +} + +func (q *MatchQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + analyzerName := "" + if q.Analyzer != "" { + analyzerName = q.Analyzer + } else { + analyzerName = m.AnalyzerNameForPath(field) + } + analyzer := m.AnalyzerNamed(analyzerName) + + if analyzer == nil { + return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) + } + + tokens := analyzer.Analyze([]byte(q.Match)) + if len(tokens) > 0 { + + tqs := make([]Query, len(tokens)) + if q.Fuzziness != 0 { + for i, token := range tokens { + query := NewFuzzyQuery(string(token.Term)) + query.SetFuzziness(q.Fuzziness) + query.SetPrefix(q.Prefix) + query.SetField(field) + query.SetBoost(q.BoostVal.Value()) + tqs[i] = query + } + } else { + for i, token := range tokens { + tq := NewTermQuery(string(token.Term)) + tq.SetField(field) + tq.SetBoost(q.BoostVal.Value()) + tqs[i] = tq + } + } + + switch q.Operator { + case MatchQueryOperatorOr: + shouldQuery := NewDisjunctionQuery(tqs) + shouldQuery.SetMin(1) + shouldQuery.SetBoost(q.BoostVal.Value()) + return shouldQuery.Searcher(i, m, options) + + case MatchQueryOperatorAnd: + mustQuery := NewConjunctionQuery(tqs) + mustQuery.SetBoost(q.BoostVal.Value()) + return mustQuery.Searcher(i, m, options) + + default: + return nil, fmt.Errorf("unhandled operator %d", q.Operator) + } + } + noneQuery := NewMatchNoneQuery() + return noneQuery.Searcher(i, m, options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_all.go b/vendor/github.com/blevesearch/bleve/search/query/match_all.go new file mode 100644 index 00000000..7fbe1f99 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/match_all.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type MatchAllQuery struct { + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewMatchAllQuery creates a Query which will +// match all documents in the index. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +func (q *MatchAllQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchAllQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchAllQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMatchAllSearcher(i, q.BoostVal.Value(), options) +} + +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + tmp := map[string]interface{}{ + "boost": q.BoostVal, + "match_all": map[string]interface{}{}, + } + return json.Marshal(tmp) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_none.go b/vendor/github.com/blevesearch/bleve/search/query/match_none.go new file mode 100644 index 00000000..dc2ea780 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/match_none.go @@ -0,0 +1,55 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type MatchNoneQuery struct { + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewMatchNoneQuery creates a Query which will not +// match any documents in the index. +func NewMatchNoneQuery() *MatchNoneQuery { + return &MatchNoneQuery{} +} + +func (q *MatchNoneQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchNoneQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchNoneQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMatchNoneSearcher(i) +} + +func (q *MatchNoneQuery) MarshalJSON() ([]byte, error) { + tmp := map[string]interface{}{ + "boost": q.BoostVal, + "match_none": map[string]interface{}{}, + } + return json.Marshal(tmp) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/match_phrase.go b/vendor/github.com/blevesearch/bleve/search/query/match_phrase.go new file mode 100644 index 00000000..51be3552 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/match_phrase.go @@ -0,0 +1,113 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "fmt" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" +) + +type MatchPhraseQuery struct { + MatchPhrase string `json:"match_phrase"` + FieldVal string `json:"field,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewMatchPhraseQuery creates a new Query object +// for matching phrases in the index. +// An Analyzer is chosen based on the field. +// Input text is analyzed using this analyzer. +// Token terms resulting from this analysis are +// used to build a search phrase. Result documents +// must match this phrase. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMatchPhraseQuery(matchPhrase string) *MatchPhraseQuery { + return &MatchPhraseQuery{ + MatchPhrase: matchPhrase, + } +} + +func (q *MatchPhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MatchPhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MatchPhraseQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *MatchPhraseQuery) Field() string { + return q.FieldVal +} + +func (q *MatchPhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + + analyzerName := "" + if q.Analyzer != "" { + analyzerName = q.Analyzer + } else { + analyzerName = m.AnalyzerNameForPath(field) + } + analyzer := m.AnalyzerNamed(analyzerName) + if analyzer == nil { + return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) + } + + tokens := analyzer.Analyze([]byte(q.MatchPhrase)) + if len(tokens) > 0 { + phrase := tokenStreamToPhrase(tokens) + phraseQuery := NewMultiPhraseQuery(phrase, field) + phraseQuery.SetBoost(q.BoostVal.Value()) + return phraseQuery.Searcher(i, m, options) + } + noneQuery := NewMatchNoneQuery() + return noneQuery.Searcher(i, m, options) +} + +func tokenStreamToPhrase(tokens analysis.TokenStream) [][]string { + firstPosition := int(^uint(0) >> 1) + lastPosition := 0 + for _, token := range tokens { + if token.Position < firstPosition { + firstPosition = token.Position + } + if token.Position > lastPosition { + lastPosition = token.Position + } + } + phraseLen := lastPosition - firstPosition + 1 + if phraseLen > 0 { + rv := make([][]string, phraseLen) + for _, token := range tokens { + pos := token.Position - firstPosition + rv[pos] = append(rv[pos], string(token.Term)) + } + return rv + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go b/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go new file mode 100644 index 00000000..8a7c9b6a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/multi_phrase.go @@ -0,0 +1,80 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type MultiPhraseQuery struct { + Terms [][]string `json:"terms"` + Field string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewMultiPhraseQuery creates a new Query for finding +// term phrases in the index. +// It is like PhraseQuery, but each position in the +// phrase may be satisfied by a list of terms +// as opposed to just one. +// At least one of the terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewMultiPhraseQuery(terms [][]string, field string) *MultiPhraseQuery { + return &MultiPhraseQuery{ + Terms: terms, + Field: field, + } +} + +func (q *MultiPhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *MultiPhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *MultiPhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewMultiPhraseSearcher(i, q.Terms, q.Field, options) +} + +func (q *MultiPhraseQuery) Validate() error { + if len(q.Terms) < 1 { + return fmt.Errorf("phrase query must contain at least one term") + } + return nil +} + +func (q *MultiPhraseQuery) UnmarshalJSON(data []byte) error { + type _mphraseQuery MultiPhraseQuery + tmp := _mphraseQuery{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + q.Terms = tmp.Terms + q.Field = tmp.Field + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/numeric_range.go b/vendor/github.com/blevesearch/bleve/search/query/numeric_range.go new file mode 100644 index 00000000..ea3f0688 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/numeric_range.go @@ -0,0 +1,87 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type NumericRangeQuery struct { + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` + InclusiveMin *bool `json:"inclusive_min,omitempty"` + InclusiveMax *bool `json:"inclusive_max,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewNumericRangeQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewNumericRangeQuery(min, max *float64) *NumericRangeQuery { + return NewNumericRangeInclusiveQuery(min, max, nil, nil) +} + +// NewNumericRangeInclusiveQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewNumericRangeInclusiveQuery(min, max *float64, minInclusive, maxInclusive *bool) *NumericRangeQuery { + return &NumericRangeQuery{ + Min: min, + Max: max, + InclusiveMin: minInclusive, + InclusiveMax: maxInclusive, + } +} + +func (q *NumericRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *NumericRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *NumericRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *NumericRangeQuery) Field() string { + return q.FieldVal +} + +func (q *NumericRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + return searcher.NewNumericRangeSearcher(i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) +} + +func (q *NumericRangeQuery) Validate() error { + if q.Min == nil && q.Min == q.Max { + return fmt.Errorf("numeric range query must specify min or max") + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/phrase.go b/vendor/github.com/blevesearch/bleve/search/query/phrase.go new file mode 100644 index 00000000..dff1a02d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/phrase.go @@ -0,0 +1,77 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type PhraseQuery struct { + Terms []string `json:"terms"` + Field string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewPhraseQuery creates a new Query for finding +// exact term phrases in the index. +// The provided terms must exist in the correct +// order, at the correct index offsets, in the +// specified field. Queried field must have been indexed with +// IncludeTermVectors set to true. +func NewPhraseQuery(terms []string, field string) *PhraseQuery { + return &PhraseQuery{ + Terms: terms, + Field: field, + } +} + +func (q *PhraseQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *PhraseQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *PhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + return searcher.NewPhraseSearcher(i, q.Terms, q.Field, options) +} + +func (q *PhraseQuery) Validate() error { + if len(q.Terms) < 1 { + return fmt.Errorf("phrase query must contain at least one term") + } + return nil +} + +func (q *PhraseQuery) UnmarshalJSON(data []byte) error { + type _phraseQuery PhraseQuery + tmp := _phraseQuery{} + err := json.Unmarshal(data, &tmp) + if err != nil { + return err + } + q.Terms = tmp.Terms + q.Field = tmp.Field + q.BoostVal = tmp.BoostVal + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/prefix.go b/vendor/github.com/blevesearch/bleve/search/query/prefix.go new file mode 100644 index 00000000..4f5be2b2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/prefix.go @@ -0,0 +1,62 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type PrefixQuery struct { + Prefix string `json:"prefix"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewPrefixQuery creates a new Query which finds +// documents containing terms that start with the +// specified prefix. +func NewPrefixQuery(prefix string) *PrefixQuery { + return &PrefixQuery{ + Prefix: prefix, + } +} + +func (q *PrefixQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *PrefixQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *PrefixQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *PrefixQuery) Field() string { + return q.FieldVal +} + +func (q *PrefixQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + return searcher.NewTermPrefixSearcher(i, q.Prefix, field, q.BoostVal.Value(), options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/query.go b/vendor/github.com/blevesearch/bleve/search/query/query.go new file mode 100644 index 00000000..1b0d94c0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/query.go @@ -0,0 +1,356 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" +) + +var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) + +// SetLog sets the logger used for logging +// by default log messages are sent to ioutil.Discard +func SetLog(l *log.Logger) { + logger = l +} + +// A Query represents a description of the type +// and parameters for a query into the index. +type Query interface { + Searcher(i index.IndexReader, m mapping.IndexMapping, + options search.SearcherOptions) (search.Searcher, error) +} + +// A BoostableQuery represents a Query which can be boosted +// relative to other queries. +type BoostableQuery interface { + Query + SetBoost(b float64) + Boost() float64 +} + +// A FieldableQuery represents a Query which can be restricted +// to a single field. +type FieldableQuery interface { + Query + SetField(f string) + Field() string +} + +// A ValidatableQuery represents a Query which can be validated +// prior to execution. +type ValidatableQuery interface { + Query + Validate() error +} + +// ParseQuery deserializes a JSON representation of +// a Query object. +func ParseQuery(input []byte) (Query, error) { + var tmp map[string]interface{} + err := json.Unmarshal(input, &tmp) + if err != nil { + return nil, err + } + _, isMatchQuery := tmp["match"] + _, hasFuzziness := tmp["fuzziness"] + if hasFuzziness && !isMatchQuery { + var rv FuzzyQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, isTermQuery := tmp["term"] + if isTermQuery { + var rv TermQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + if isMatchQuery { + var rv MatchQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, isMatchPhraseQuery := tmp["match_phrase"] + if isMatchPhraseQuery { + var rv MatchPhraseQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMust := tmp["must"] + _, hasShould := tmp["should"] + _, hasMustNot := tmp["must_not"] + if hasMust || hasShould || hasMustNot { + var rv BooleanQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasTerms := tmp["terms"] + if hasTerms { + var rv PhraseQuery + err := json.Unmarshal(input, &rv) + if err != nil { + // now try multi-phrase + var rv2 MultiPhraseQuery + err = json.Unmarshal(input, &rv2) + if err != nil { + return nil, err + } + return &rv2, nil + } + return &rv, nil + } + _, hasConjuncts := tmp["conjuncts"] + if hasConjuncts { + var rv ConjunctionQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDisjuncts := tmp["disjuncts"] + if hasDisjuncts { + var rv DisjunctionQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + + _, hasSyntaxQuery := tmp["query"] + if hasSyntaxQuery { + var rv QueryStringQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMin := tmp["min"].(float64) + _, hasMax := tmp["max"].(float64) + if hasMin || hasMax { + var rv NumericRangeQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMinStr := tmp["min"].(string) + _, hasMaxStr := tmp["max"].(string) + if hasMinStr || hasMaxStr { + var rv TermRangeQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasStart := tmp["start"] + _, hasEnd := tmp["end"] + if hasStart || hasEnd { + var rv DateRangeQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasPrefix := tmp["prefix"] + if hasPrefix { + var rv PrefixQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasRegexp := tmp["regexp"] + if hasRegexp { + var rv RegexpQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasWildcard := tmp["wildcard"] + if hasWildcard { + var rv WildcardQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMatchAll := tmp["match_all"] + if hasMatchAll { + var rv MatchAllQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasMatchNone := tmp["match_none"] + if hasMatchNone { + var rv MatchNoneQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDocIds := tmp["ids"] + if hasDocIds { + var rv DocIDQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasBool := tmp["bool"] + if hasBool { + var rv BoolFieldQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasTopLeft := tmp["top_left"] + _, hasBottomRight := tmp["bottom_right"] + if hasTopLeft && hasBottomRight { + var rv GeoBoundingBoxQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + _, hasDistance := tmp["distance"] + if hasDistance { + var rv GeoDistanceQuery + err := json.Unmarshal(input, &rv) + if err != nil { + return nil, err + } + return &rv, nil + } + return nil, fmt.Errorf("unknown query type") +} + +// expandQuery traverses the input query tree and returns a new tree where +// query string queries have been expanded into base queries. Returned tree may +// reference queries from the input tree or new queries. +func expandQuery(m mapping.IndexMapping, query Query) (Query, error) { + var expand func(query Query) (Query, error) + var expandSlice func(queries []Query) ([]Query, error) + + expandSlice = func(queries []Query) ([]Query, error) { + expanded := []Query{} + for _, q := range queries { + exp, err := expand(q) + if err != nil { + return nil, err + } + expanded = append(expanded, exp) + } + return expanded, nil + } + + expand = func(query Query) (Query, error) { + switch query.(type) { + case *QueryStringQuery: + q := query.(*QueryStringQuery) + parsed, err := parseQuerySyntax(q.Query) + if err != nil { + return nil, fmt.Errorf("could not parse '%s': %s", q.Query, err) + } + return expand(parsed) + case *ConjunctionQuery: + q := *query.(*ConjunctionQuery) + children, err := expandSlice(q.Conjuncts) + if err != nil { + return nil, err + } + q.Conjuncts = children + return &q, nil + case *DisjunctionQuery: + q := *query.(*DisjunctionQuery) + children, err := expandSlice(q.Disjuncts) + if err != nil { + return nil, err + } + q.Disjuncts = children + return &q, nil + case *BooleanQuery: + q := *query.(*BooleanQuery) + var err error + q.Must, err = expand(q.Must) + if err != nil { + return nil, err + } + q.Should, err = expand(q.Should) + if err != nil { + return nil, err + } + q.MustNot, err = expand(q.MustNot) + if err != nil { + return nil, err + } + return &q, nil + default: + return query, nil + } + } + return expand(query) +} + +// DumpQuery returns a string representation of the query tree, where query +// string queries have been expanded into base queries. The output format is +// meant for debugging purpose and may change in the future. +func DumpQuery(m mapping.IndexMapping, query Query) (string, error) { + q, err := expandQuery(m, query) + if err != nil { + return "", err + } + data, err := json.MarshalIndent(q, "", " ") + return string(data), err +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.go b/vendor/github.com/blevesearch/bleve/search/query/query_string.go new file mode 100644 index 00000000..7d93d34b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string.go @@ -0,0 +1,63 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" +) + +type QueryStringQuery struct { + Query string `json:"query"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewQueryStringQuery creates a new Query used for +// finding documents that satisfy a query string. The +// query string is a small query language for humans. +func NewQueryStringQuery(query string) *QueryStringQuery { + return &QueryStringQuery{ + Query: query, + } +} + +func (q *QueryStringQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *QueryStringQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *QueryStringQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + newQuery, err := parseQuerySyntax(q.Query) + if err != nil { + return nil, err + } + return newQuery.Searcher(i, m, options) +} + +func (q *QueryStringQuery) Validate() error { + newQuery, err := parseQuerySyntax(q.Query) + if err != nil { + return err + } + if newQuery, ok := newQuery.(ValidatableQuery); ok { + return newQuery.Validate() + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.y b/vendor/github.com/blevesearch/bleve/search/query/query_string.y new file mode 100644 index 00000000..d3e5ac9d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string.y @@ -0,0 +1,328 @@ +%{ +package query +import ( + "fmt" + "strconv" + "strings" + "time" +) + +func logDebugGrammar(format string, v ...interface{}) { + if debugParser { + logger.Printf(format, v...) + } +} +%} + +%union { +s string +n int +f float64 +q Query +pf *float64} + +%token tSTRING tPHRASE tPLUS tMINUS tCOLON tBOOST tNUMBER tSTRING tGREATER tLESS +tEQUAL tTILDE + +%type tSTRING +%type tPHRASE +%type tNUMBER +%type posOrNegNumber +%type tTILDE +%type tBOOST +%type searchBase +%type searchSuffix +%type searchPrefix + +%% + +input: +searchParts { + logDebugGrammar("INPUT") +}; + +searchParts: +searchPart searchParts { + logDebugGrammar("SEARCH PARTS") +} +| +searchPart { + logDebugGrammar("SEARCH PART") +}; + +searchPart: +searchPrefix searchBase searchSuffix { + query := $2 + if $3 != nil { + if query, ok := query.(BoostableQuery); ok { + query.SetBoost(*$3) + } + } + switch($1) { + case queryShould: + yylex.(*lexerWrapper).query.AddShould(query) + case queryMust: + yylex.(*lexerWrapper).query.AddMust(query) + case queryMustNot: + yylex.(*lexerWrapper).query.AddMustNot(query) + } +}; + + +searchPrefix: +/* empty */ { + $$ = queryShould +} +| +tPLUS { + logDebugGrammar("PLUS") + $$ = queryMust +} +| +tMINUS { + logDebugGrammar("MINUS") + $$ = queryMustNot +}; + +searchBase: +tSTRING { + str := $1 + logDebugGrammar("STRING - %s", str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1:len(str)-1]) + } else if strings.ContainsAny(str, "*?"){ + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + $$ = q +} +| +tSTRING tTILDE { + str := $1 + fuzziness, err := strconv.ParseFloat($2, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FUZZY STRING - %s %f", str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + $$ = q +} +| +tSTRING tCOLON tSTRING tTILDE { + field := $1 + str := $3 + fuzziness, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FIELD - %s FUZZY STRING - %s %f", field, str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + q.SetField(field) + $$ = q +} +| +tNUMBER { + str := $1 + logDebugGrammar("STRING - %s", str) + q1 := NewMatchQuery(str) + val, err := strconv.ParseFloat($1, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q := NewDisjunctionQuery([]Query{q1,q2}) + q.queryStringMode = true + $$ = q +} +| +tPHRASE { + phrase := $1 + logDebugGrammar("PHRASE - %s", phrase) + q := NewMatchPhraseQuery(phrase) + $$ = q +} +| +tSTRING tCOLON tSTRING { + field := $1 + str := $3 + logDebugGrammar("FIELD - %s STRING - %s", field, str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1:len(str)-1]) + } else if strings.ContainsAny(str, "*?"){ + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON posOrNegNumber { + field := $1 + str := $3 + logDebugGrammar("FIELD - %s STRING - %s", field, str) + q1 := NewMatchQuery(str) + q1.SetField(field) + val, err := strconv.ParseFloat($3, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q2.SetField(field) + q := NewDisjunctionQuery([]Query{q1,q2}) + q.queryStringMode = true + $$ = q +} +| +tSTRING tCOLON tPHRASE { + field := $1 + phrase := $3 + logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) + q := NewMatchPhraseQuery(phrase) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tGREATER posOrNegNumber { + field := $1 + min, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := false + logDebugGrammar("FIELD - GREATER THAN %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL posOrNegNumber { + field := $1 + min, err := strconv.ParseFloat($5, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := true + logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tLESS posOrNegNumber { + field := $1 + max, err := strconv.ParseFloat($4, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := false + logDebugGrammar("FIELD - LESS THAN %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL posOrNegNumber { + field := $1 + max, err := strconv.ParseFloat($5, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := true + logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tGREATER tPHRASE { + field := $1 + minInclusive := false + phrase := $4 + + logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tGREATER tEQUAL tPHRASE { + field := $1 + minInclusive := true + phrase := $5 + + logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tLESS tPHRASE { + field := $1 + maxInclusive := false + phrase := $4 + + logDebugGrammar("FIELD - LESS THAN DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + $$ = q +} +| +tSTRING tCOLON tLESS tEQUAL tPHRASE { + field := $1 + maxInclusive := true + phrase := $5 + + logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + $$ = q +}; + +searchSuffix: +/* empty */ { + $$ = nil +} +| +tBOOST { + $$ = nil + boost, err := strconv.ParseFloat($1, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid boost value: %v", err)) + } else { + $$ = &boost + } + logDebugGrammar("BOOST %f", boost) +}; + +posOrNegNumber: +tNUMBER { + $$ = $1 +} +| +tMINUS tNUMBER { + $$ = "-" + $2 +}; diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.y.go b/vendor/github.com/blevesearch/bleve/search/query/query_string.y.go new file mode 100644 index 00000000..ac2d3222 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string.y.go @@ -0,0 +1,815 @@ +package query + +import __yyfmt__ "fmt" + +//line query_string.y:2 +import ( + "fmt" + "strconv" + "strings" + "time" +) + +func logDebugGrammar(format string, v ...interface{}) { + if debugParser { + logger.Printf(format, v...) + } +} + +//line query_string.y:17 +type yySymType struct { + yys int + s string + n int + f float64 + q Query + pf *float64 +} + +const tSTRING = 57346 +const tPHRASE = 57347 +const tPLUS = 57348 +const tMINUS = 57349 +const tCOLON = 57350 +const tBOOST = 57351 +const tNUMBER = 57352 +const tGREATER = 57353 +const tLESS = 57354 +const tEQUAL = 57355 +const tTILDE = 57356 + +var yyToknames = [...]string{ + "$end", + "error", + "$unk", + "tSTRING", + "tPHRASE", + "tPLUS", + "tMINUS", + "tCOLON", + "tBOOST", + "tNUMBER", + "tGREATER", + "tLESS", + "tEQUAL", + "tTILDE", +} +var yyStatenames = [...]string{} + +const yyEofCode = 1 +const yyErrCode = 2 +const yyInitialStackSize = 16 + +//line yacctab:1 +var yyExca = [...]int{ + -1, 1, + 1, -1, + -2, 0, + -1, 3, + 1, 3, + -2, 5, +} + +const yyNprod = 28 +const yyPrivate = 57344 + +var yyTokenNames []string +var yyStates []string + +const yyLast = 42 + +var yyAct = [...]int{ + + 17, 16, 18, 23, 22, 30, 3, 21, 19, 20, + 29, 26, 22, 22, 1, 21, 21, 15, 28, 25, + 24, 27, 34, 14, 22, 13, 31, 21, 32, 33, + 22, 9, 11, 21, 5, 6, 2, 10, 4, 12, + 7, 8, +} +var yyPact = [...]int{ + + 28, -1000, -1000, 28, 27, -1000, -1000, -1000, 16, 9, + -1000, -1000, -1000, -1000, -1000, -3, -11, -1000, -1000, 6, + 5, -1000, -5, -1000, -1000, 23, -1000, -1000, 17, -1000, + -1000, -1000, -1000, -1000, -1000, +} +var yyPgo = [...]int{ + + 0, 0, 41, 39, 38, 14, 36, 6, +} +var yyR1 = [...]int{ + + 0, 5, 6, 6, 7, 4, 4, 4, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 3, 3, 1, 1, +} +var yyR2 = [...]int{ + + 0, 1, 2, 1, 3, 0, 1, 1, 1, 2, + 4, 1, 1, 3, 3, 3, 4, 5, 4, 5, + 4, 5, 4, 5, 0, 1, 1, 2, +} +var yyChk = [...]int{ + + -1000, -5, -6, -7, -4, 6, 7, -6, -2, 4, + 10, 5, -3, 9, 14, 8, 4, -1, 5, 11, + 12, 10, 7, 14, -1, 13, 5, -1, 13, 5, + 10, -1, 5, -1, 5, +} +var yyDef = [...]int{ + + 5, -2, 1, -2, 0, 6, 7, 2, 24, 8, + 11, 12, 4, 25, 9, 0, 13, 14, 15, 0, + 0, 26, 0, 10, 16, 0, 20, 18, 0, 22, + 27, 17, 21, 19, 23, +} +var yyTok1 = [...]int{ + + 1, +} +var yyTok2 = [...]int{ + + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, +} +var yyTok3 = [...]int{ + 0, +} + +var yyErrorMessages = [...]struct { + state int + token int + msg string +}{} + +//line yaccpar:1 + +/* parser for yacc output */ + +var ( + yyDebug = 0 + yyErrorVerbose = false +) + +type yyLexer interface { + Lex(lval *yySymType) int + Error(s string) +} + +type yyParser interface { + Parse(yyLexer) int + Lookahead() int +} + +type yyParserImpl struct { + lval yySymType + stack [yyInitialStackSize]yySymType + char int +} + +func (p *yyParserImpl) Lookahead() int { + return p.char +} + +func yyNewParser() yyParser { + return &yyParserImpl{} +} + +const yyFlag = -1000 + +func yyTokname(c int) string { + if c >= 1 && c-1 < len(yyToknames) { + if yyToknames[c-1] != "" { + return yyToknames[c-1] + } + } + return __yyfmt__.Sprintf("tok-%v", c) +} + +func yyStatname(s int) string { + if s >= 0 && s < len(yyStatenames) { + if yyStatenames[s] != "" { + return yyStatenames[s] + } + } + return __yyfmt__.Sprintf("state-%v", s) +} + +func yyErrorMessage(state, lookAhead int) string { + const TOKSTART = 4 + + if !yyErrorVerbose { + return "syntax error" + } + + for _, e := range yyErrorMessages { + if e.state == state && e.token == lookAhead { + return "syntax error: " + e.msg + } + } + + res := "syntax error: unexpected " + yyTokname(lookAhead) + + // To match Bison, suggest at most four expected tokens. + expected := make([]int, 0, 4) + + // Look for shiftable tokens. + base := yyPact[state] + for tok := TOKSTART; tok-1 < len(yyToknames); tok++ { + if n := base + tok; n >= 0 && n < yyLast && yyChk[yyAct[n]] == tok { + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + } + + if yyDef[state] == -2 { + i := 0 + for yyExca[i] != -1 || yyExca[i+1] != state { + i += 2 + } + + // Look for tokens that we accept or reduce. + for i += 2; yyExca[i] >= 0; i += 2 { + tok := yyExca[i] + if tok < TOKSTART || yyExca[i+1] == 0 { + continue + } + if len(expected) == cap(expected) { + return res + } + expected = append(expected, tok) + } + + // If the default action is to accept or reduce, give up. + if yyExca[i+1] != 0 { + return res + } + } + + for i, tok := range expected { + if i == 0 { + res += ", expecting " + } else { + res += " or " + } + res += yyTokname(tok) + } + return res +} + +func yylex1(lex yyLexer, lval *yySymType) (char, token int) { + token = 0 + char = lex.Lex(lval) + if char <= 0 { + token = yyTok1[0] + goto out + } + if char < len(yyTok1) { + token = yyTok1[char] + goto out + } + if char >= yyPrivate { + if char < yyPrivate+len(yyTok2) { + token = yyTok2[char-yyPrivate] + goto out + } + } + for i := 0; i < len(yyTok3); i += 2 { + token = yyTok3[i+0] + if token == char { + token = yyTok3[i+1] + goto out + } + } + +out: + if token == 0 { + token = yyTok2[1] /* unknown char */ + } + if yyDebug >= 3 { + __yyfmt__.Printf("lex %s(%d)\n", yyTokname(token), uint(char)) + } + return char, token +} + +func yyParse(yylex yyLexer) int { + return yyNewParser().Parse(yylex) +} + +func (yyrcvr *yyParserImpl) Parse(yylex yyLexer) int { + var yyn int + var yyVAL yySymType + var yyDollar []yySymType + _ = yyDollar // silence set and not used + yyS := yyrcvr.stack[:] + + Nerrs := 0 /* number of errors */ + Errflag := 0 /* error recovery flag */ + yystate := 0 + yyrcvr.char = -1 + yytoken := -1 // yyrcvr.char translated into internal numbering + defer func() { + // Make sure we report no lookahead when not parsing. + yystate = -1 + yyrcvr.char = -1 + yytoken = -1 + }() + yyp := -1 + goto yystack + +ret0: + return 0 + +ret1: + return 1 + +yystack: + /* put a state and value onto the stack */ + if yyDebug >= 4 { + __yyfmt__.Printf("char %v in %v\n", yyTokname(yytoken), yyStatname(yystate)) + } + + yyp++ + if yyp >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyS[yyp] = yyVAL + yyS[yyp].yys = yystate + +yynewstate: + yyn = yyPact[yystate] + if yyn <= yyFlag { + goto yydefault /* simple state */ + } + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + yyn += yytoken + if yyn < 0 || yyn >= yyLast { + goto yydefault + } + yyn = yyAct[yyn] + if yyChk[yyn] == yytoken { /* valid shift */ + yyrcvr.char = -1 + yytoken = -1 + yyVAL = yyrcvr.lval + yystate = yyn + if Errflag > 0 { + Errflag-- + } + goto yystack + } + +yydefault: + /* default state action */ + yyn = yyDef[yystate] + if yyn == -2 { + if yyrcvr.char < 0 { + yyrcvr.char, yytoken = yylex1(yylex, &yyrcvr.lval) + } + + /* look through exception table */ + xi := 0 + for { + if yyExca[xi+0] == -1 && yyExca[xi+1] == yystate { + break + } + xi += 2 + } + for xi += 2; ; xi += 2 { + yyn = yyExca[xi+0] + if yyn < 0 || yyn == yytoken { + break + } + } + yyn = yyExca[xi+1] + if yyn < 0 { + goto ret0 + } + } + if yyn == 0 { + /* error ... attempt to resume parsing */ + switch Errflag { + case 0: /* brand new error */ + yylex.Error(yyErrorMessage(yystate, yytoken)) + Nerrs++ + if yyDebug >= 1 { + __yyfmt__.Printf("%s", yyStatname(yystate)) + __yyfmt__.Printf(" saw %s\n", yyTokname(yytoken)) + } + fallthrough + + case 1, 2: /* incompletely recovered error ... try again */ + Errflag = 3 + + /* find a state where "error" is a legal shift action */ + for yyp >= 0 { + yyn = yyPact[yyS[yyp].yys] + yyErrCode + if yyn >= 0 && yyn < yyLast { + yystate = yyAct[yyn] /* simulate a shift of "error" */ + if yyChk[yystate] == yyErrCode { + goto yystack + } + } + + /* the current p has no shift on "error", pop stack */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys) + } + yyp-- + } + /* there is no state on the stack with an error shift ... abort */ + goto ret1 + + case 3: /* no shift yet; clobber input char */ + if yyDebug >= 2 { + __yyfmt__.Printf("error recovery discards %s\n", yyTokname(yytoken)) + } + if yytoken == yyEofCode { + goto ret1 + } + yyrcvr.char = -1 + yytoken = -1 + goto yynewstate /* try again in the same state */ + } + } + + /* reduction by production yyn */ + if yyDebug >= 2 { + __yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate)) + } + + yynt := yyn + yypt := yyp + _ = yypt // guard against "declared and not used" + + yyp -= yyR2[yyn] + // yyp is now the index of $0. Perform the default action. Iff the + // reduced production is ε, $1 is possibly out of range. + if yyp+1 >= len(yyS) { + nyys := make([]yySymType, len(yyS)*2) + copy(nyys, yyS) + yyS = nyys + } + yyVAL = yyS[yyp+1] + + /* consult goto table to find next state */ + yyn = yyR1[yyn] + yyg := yyPgo[yyn] + yyj := yyg + yyS[yyp].yys + 1 + + if yyj >= yyLast { + yystate = yyAct[yyg] + } else { + yystate = yyAct[yyj] + if yyChk[yystate] != -yyn { + yystate = yyAct[yyg] + } + } + // dummy call; replaced with literal code + switch yynt { + + case 1: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:40 + { + logDebugGrammar("INPUT") + } + case 2: + yyDollar = yyS[yypt-2 : yypt+1] + //line query_string.y:45 + { + logDebugGrammar("SEARCH PARTS") + } + case 3: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:49 + { + logDebugGrammar("SEARCH PART") + } + case 4: + yyDollar = yyS[yypt-3 : yypt+1] + //line query_string.y:54 + { + query := yyDollar[2].q + if yyDollar[3].pf != nil { + if query, ok := query.(BoostableQuery); ok { + query.SetBoost(*yyDollar[3].pf) + } + } + switch yyDollar[1].n { + case queryShould: + yylex.(*lexerWrapper).query.AddShould(query) + case queryMust: + yylex.(*lexerWrapper).query.AddMust(query) + case queryMustNot: + yylex.(*lexerWrapper).query.AddMustNot(query) + } + } + case 5: + yyDollar = yyS[yypt-0 : yypt+1] + //line query_string.y:73 + { + yyVAL.n = queryShould + } + case 6: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:77 + { + logDebugGrammar("PLUS") + yyVAL.n = queryMust + } + case 7: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:82 + { + logDebugGrammar("MINUS") + yyVAL.n = queryMustNot + } + case 8: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:88 + { + str := yyDollar[1].s + logDebugGrammar("STRING - %s", str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1 : len(str)-1]) + } else if strings.ContainsAny(str, "*?") { + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + yyVAL.q = q + } + case 9: + yyDollar = yyS[yypt-2 : yypt+1] + //line query_string.y:102 + { + str := yyDollar[1].s + fuzziness, err := strconv.ParseFloat(yyDollar[2].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FUZZY STRING - %s %f", str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + yyVAL.q = q + } + case 10: + yyDollar = yyS[yypt-4 : yypt+1] + //line query_string.y:114 + { + field := yyDollar[1].s + str := yyDollar[3].s + fuzziness, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid fuzziness value: %v", err)) + } + logDebugGrammar("FIELD - %s FUZZY STRING - %s %f", field, str, fuzziness) + q := NewMatchQuery(str) + q.SetFuzziness(int(fuzziness)) + q.SetField(field) + yyVAL.q = q + } + case 11: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:128 + { + str := yyDollar[1].s + logDebugGrammar("STRING - %s", str) + q1 := NewMatchQuery(str) + val, err := strconv.ParseFloat(yyDollar[1].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q := NewDisjunctionQuery([]Query{q1, q2}) + q.queryStringMode = true + yyVAL.q = q + } + case 12: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:143 + { + phrase := yyDollar[1].s + logDebugGrammar("PHRASE - %s", phrase) + q := NewMatchPhraseQuery(phrase) + yyVAL.q = q + } + case 13: + yyDollar = yyS[yypt-3 : yypt+1] + //line query_string.y:150 + { + field := yyDollar[1].s + str := yyDollar[3].s + logDebugGrammar("FIELD - %s STRING - %s", field, str) + var q FieldableQuery + if strings.HasPrefix(str, "/") && strings.HasSuffix(str, "/") { + q = NewRegexpQuery(str[1 : len(str)-1]) + } else if strings.ContainsAny(str, "*?") { + q = NewWildcardQuery(str) + } else { + q = NewMatchQuery(str) + } + q.SetField(field) + yyVAL.q = q + } + case 14: + yyDollar = yyS[yypt-3 : yypt+1] + //line query_string.y:166 + { + field := yyDollar[1].s + str := yyDollar[3].s + logDebugGrammar("FIELD - %s STRING - %s", field, str) + q1 := NewMatchQuery(str) + q1.SetField(field) + val, err := strconv.ParseFloat(yyDollar[3].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + inclusive := true + q2 := NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive) + q2.SetField(field) + q := NewDisjunctionQuery([]Query{q1, q2}) + q.queryStringMode = true + yyVAL.q = q + } + case 15: + yyDollar = yyS[yypt-3 : yypt+1] + //line query_string.y:184 + { + field := yyDollar[1].s + phrase := yyDollar[3].s + logDebugGrammar("FIELD - %s PHRASE - %s", field, phrase) + q := NewMatchPhraseQuery(phrase) + q.SetField(field) + yyVAL.q = q + } + case 16: + yyDollar = yyS[yypt-4 : yypt+1] + //line query_string.y:193 + { + field := yyDollar[1].s + min, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := false + logDebugGrammar("FIELD - GREATER THAN %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 17: + yyDollar = yyS[yypt-5 : yypt+1] + //line query_string.y:206 + { + field := yyDollar[1].s + min, err := strconv.ParseFloat(yyDollar[5].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + minInclusive := true + logDebugGrammar("FIELD - GREATER THAN OR EQUAL %f", min) + q := NewNumericRangeInclusiveQuery(&min, nil, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 18: + yyDollar = yyS[yypt-4 : yypt+1] + //line query_string.y:219 + { + field := yyDollar[1].s + max, err := strconv.ParseFloat(yyDollar[4].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := false + logDebugGrammar("FIELD - LESS THAN %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 19: + yyDollar = yyS[yypt-5 : yypt+1] + //line query_string.y:232 + { + field := yyDollar[1].s + max, err := strconv.ParseFloat(yyDollar[5].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("error parsing number: %v", err)) + } + maxInclusive := true + logDebugGrammar("FIELD - LESS THAN OR EQUAL %f", max) + q := NewNumericRangeInclusiveQuery(nil, &max, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 20: + yyDollar = yyS[yypt-4 : yypt+1] + //line query_string.y:245 + { + field := yyDollar[1].s + minInclusive := false + phrase := yyDollar[4].s + + logDebugGrammar("FIELD - GREATER THAN DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 21: + yyDollar = yyS[yypt-5 : yypt+1] + //line query_string.y:260 + { + field := yyDollar[1].s + minInclusive := true + phrase := yyDollar[5].s + + logDebugGrammar("FIELD - GREATER THAN OR EQUAL DATE %s", phrase) + minTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(minTime, time.Time{}, &minInclusive, nil) + q.SetField(field) + yyVAL.q = q + } + case 22: + yyDollar = yyS[yypt-4 : yypt+1] + //line query_string.y:275 + { + field := yyDollar[1].s + maxInclusive := false + phrase := yyDollar[4].s + + logDebugGrammar("FIELD - LESS THAN DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 23: + yyDollar = yyS[yypt-5 : yypt+1] + //line query_string.y:290 + { + field := yyDollar[1].s + maxInclusive := true + phrase := yyDollar[5].s + + logDebugGrammar("FIELD - LESS THAN OR EQUAL DATE %s", phrase) + maxTime, err := queryTimeFromString(phrase) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid time: %v", err)) + } + q := NewDateRangeInclusiveQuery(time.Time{}, maxTime, nil, &maxInclusive) + q.SetField(field) + yyVAL.q = q + } + case 24: + yyDollar = yyS[yypt-0 : yypt+1] + //line query_string.y:306 + { + yyVAL.pf = nil + } + case 25: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:310 + { + yyVAL.pf = nil + boost, err := strconv.ParseFloat(yyDollar[1].s, 64) + if err != nil { + yylex.(*lexerWrapper).lex.Error(fmt.Sprintf("invalid boost value: %v", err)) + } else { + yyVAL.pf = &boost + } + logDebugGrammar("BOOST %f", boost) + } + case 26: + yyDollar = yyS[yypt-1 : yypt+1] + //line query_string.y:322 + { + yyVAL.s = yyDollar[1].s + } + case 27: + yyDollar = yyS[yypt-2 : yypt+1] + //line query_string.y:326 + { + yyVAL.s = "-" + yyDollar[2].s + } + } + goto yystack /* stack new state and value */ +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go b/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go new file mode 100644 index 00000000..9c59cedd --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go @@ -0,0 +1,322 @@ +// Copyright (c) 2016 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "bufio" + "io" + "strings" + "unicode" +) + +const reservedChars = "+-=&|>', '<', '=': + l.buf += string(next) + return singleCharOpState, true + case '^': + return inBoostState, true + case '~': + return inTildeState, true + } + + switch { + case !l.inEscape && next == '\\': + l.inEscape = true + return startState, true + case unicode.IsDigit(next): + l.buf += string(next) + return inNumOrStrState, true + case !unicode.IsSpace(next): + l.buf += string(next) + return inStrState, true + } + + // doesn't look like anything, just eat it and stay here + l.reset() + return startState, true +} + +func inPhraseState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // unterminated phrase eats the phrase + if eof { + l.Error("unterminated quote") + return nil, false + } + + // only a non-escaped " ends the phrase + if !l.inEscape && next == '"' { + // end phrase + l.nextTokenType = tPHRASE + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("PHRASE - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inPhraseState, true +} + +func singleCharOpState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + l.nextToken = &yySymType{} + + switch l.buf { + case "+": + l.nextTokenType = tPLUS + logDebugTokens("PLUS") + case "-": + l.nextTokenType = tMINUS + logDebugTokens("MINUS") + case ":": + l.nextTokenType = tCOLON + logDebugTokens("COLON") + case ">": + l.nextTokenType = tGREATER + logDebugTokens("GREATER") + case "<": + l.nextTokenType = tLESS + logDebugTokens("LESS") + case "=": + l.nextTokenType = tEQUAL + logDebugTokens("EQUAL") + } + + l.reset() + return startState, false +} + +func inBoostState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + + // only a non-escaped space ends the boost (or eof) + if eof || (!l.inEscape && next == ' ') { + // end boost + l.nextTokenType = tBOOST + if l.buf == "" { + l.buf = "1" + } + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("BOOST - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inBoostState, true +} + +func inTildeState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + + // only a non-escaped space ends the tilde (or eof) + if eof || (!l.inEscape && next == ' ') { + // end tilde + l.nextTokenType = tTILDE + if l.buf == "" { + l.buf = "1" + } + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("TILDE - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inTildeState, true +} + +func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // only a non-escaped space ends the tilde (or eof) + if eof || (!l.inEscape && next == ' ') { + // end number + l.nextTokenType = tNUMBER + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("NUMBER - '%s'", l.nextToken.s) + l.reset() + return startState, true + } else if !l.inEscape && next == '\\' { + l.inEscape = true + return inNumOrStrState, true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + // go directly to string, no successfully or unsuccessfully + // escaped string results in a valid number + return inStrState, true + } + + // see where to go + if !l.seenDot && next == '.' { + // stay in this state + l.buf += string(next) + return inNumOrStrState, true + } else if unicode.IsDigit(next) { + l.buf += string(next) + return inNumOrStrState, true + } + + // doesn't look like an number, transition + l.buf += string(next) + return inStrState, true +} + +func inStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { + // end on non-escped space, colon, tilde, boost (or eof) + if eof || (!l.inEscape && (next == ' ' || next == ':' || next == '^' || next == '~')) { + // end string + l.nextTokenType = tSTRING + l.nextToken = &yySymType{ + s: l.buf, + } + logDebugTokens("STRING - '%s'", l.nextToken.s) + l.reset() + + consumed := true + if !eof && (next == ':' || next == '^' || next == '~') { + consumed = false + } + + return startState, consumed + } else if !l.inEscape && next == '\\' { + l.inEscape = true + } else if l.inEscape { + // if in escape, end it + l.inEscape = false + l.buf += unescape(string(next)) + } else { + l.buf += string(next) + } + + return inStrState, true +} + +func logDebugTokens(format string, v ...interface{}) { + if debugLexer { + logger.Printf(format, v...) + } +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go b/vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go new file mode 100644 index 00000000..3fb7731b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string_parser.go @@ -0,0 +1,85 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// as of Go 1.8 this requires the goyacc external tool +// available from golang.org/x/tools/cmd/goyacc + +//go:generate goyacc -o query_string.y.go query_string.y +//go:generate sed -i.tmp -e 1d query_string.y.go +//go:generate rm query_string.y.go.tmp + +// note: OSX sed and gnu sed handle the -i (in-place) option differently. +// using -i.tmp works on both, at the expense of having to remove +// the unsightly .tmp files + +package query + +import ( + "fmt" + "strings" +) + +var debugParser bool +var debugLexer bool + +func parseQuerySyntax(query string) (rq Query, err error) { + if query == "" { + return NewMatchNoneQuery(), nil + } + lex := newLexerWrapper(newQueryStringLex(strings.NewReader(query))) + doParse(lex) + + if len(lex.errs) > 0 { + return nil, fmt.Errorf(strings.Join(lex.errs, "\n")) + } + return lex.query, nil +} + +func doParse(lex *lexerWrapper) { + defer func() { + r := recover() + if r != nil { + lex.errs = append(lex.errs, fmt.Sprintf("parse error: %v", r)) + } + }() + + yyParse(lex) +} + +const ( + queryShould = iota + queryMust + queryMustNot +) + +type lexerWrapper struct { + lex yyLexer + errs []string + query *BooleanQuery +} + +func newLexerWrapper(lex yyLexer) *lexerWrapper { + return &lexerWrapper{ + lex: lex, + query: NewBooleanQueryForQueryString(nil, nil, nil), + } +} + +func (l *lexerWrapper) Lex(lval *yySymType) int { + return l.lex.Lex(lval) +} + +func (l *lexerWrapper) Error(s string) { + l.errs = append(l.errs, s) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/regexp.go b/vendor/github.com/blevesearch/bleve/search/query/regexp.go new file mode 100644 index 00000000..09544fcf --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/regexp.go @@ -0,0 +1,96 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "regexp" + "strings" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type RegexpQuery struct { + Regexp string `json:"regexp"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + compiled *regexp.Regexp +} + +// NewRegexpQuery creates a new Query which finds +// documents containing terms that match the +// specified regular expression. The regexp pattern +// SHOULD NOT include ^ or $ modifiers, the search +// will only match entire terms even without them. +func NewRegexpQuery(regexp string) *RegexpQuery { + return &RegexpQuery{ + Regexp: regexp, + } +} + +func (q *RegexpQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *RegexpQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *RegexpQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *RegexpQuery) Field() string { + return q.FieldVal +} + +func (q *RegexpQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + err := q.compile() + if err != nil { + return nil, err + } + + return searcher.NewRegexpSearcher(i, q.compiled, field, q.BoostVal.Value(), options) +} + +func (q *RegexpQuery) Validate() error { + return q.compile() +} + +func (q *RegexpQuery) compile() error { + if q.compiled == nil { + // require that pattern NOT be anchored to start and end of term + actualRegexp := q.Regexp + if strings.HasPrefix(actualRegexp, "^") { + actualRegexp = actualRegexp[1:] // remove leading ^ + } + // do not attempt to remove trailing $, it's presence is not + // known to interfere with LiteralPrefix() the way ^ does + // and removing $ introduces possible ambiguities with escaped \$, \\$, etc + var err error + q.compiled, err = regexp.Compile(actualRegexp) + if err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/term.go b/vendor/github.com/blevesearch/bleve/search/query/term.go new file mode 100644 index 00000000..2eeb5a37 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/term.go @@ -0,0 +1,61 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type TermQuery struct { + Term string `json:"term"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewTermQuery creates a new Query for finding an +// exact term match in the index. +func NewTermQuery(term string) *TermQuery { + return &TermQuery{ + Term: term, + } +} + +func (q *TermQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *TermQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *TermQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *TermQuery) Field() string { + return q.FieldVal +} + +func (q *TermQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + return searcher.NewTermSearcher(i, q.Term, field, q.BoostVal.Value(), options) +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/term_range.go b/vendor/github.com/blevesearch/bleve/search/query/term_range.go new file mode 100644 index 00000000..8f8ca844 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/term_range.go @@ -0,0 +1,95 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "fmt" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +type TermRangeQuery struct { + Min string `json:"min,omitempty"` + Max string `json:"max,omitempty"` + InclusiveMin *bool `json:"inclusive_min,omitempty"` + InclusiveMax *bool `json:"inclusive_max,omitempty"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` +} + +// NewTermRangeQuery creates a new Query for ranges +// of text term values. +// Either, but not both endpoints can be nil. +// The minimum value is inclusive. +// The maximum value is exclusive. +func NewTermRangeQuery(min, max string) *TermRangeQuery { + return NewTermRangeInclusiveQuery(min, max, nil, nil) +} + +// NewTermRangeInclusiveQuery creates a new Query for ranges +// of numeric values. +// Either, but not both endpoints can be nil. +// Control endpoint inclusion with inclusiveMin, inclusiveMax. +func NewTermRangeInclusiveQuery(min, max string, minInclusive, maxInclusive *bool) *TermRangeQuery { + return &TermRangeQuery{ + Min: min, + Max: max, + InclusiveMin: minInclusive, + InclusiveMax: maxInclusive, + } +} + +func (q *TermRangeQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *TermRangeQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *TermRangeQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *TermRangeQuery) Field() string { + return q.FieldVal +} + +func (q *TermRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + var minTerm []byte + if q.Min != "" { + minTerm = []byte(q.Min) + } + var maxTerm []byte + if q.Max != "" { + maxTerm = []byte(q.Max) + } + return searcher.NewTermRangeSearcher(i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) +} + +func (q *TermRangeQuery) Validate() error { + if q.Min == "" && q.Min == q.Max { + return fmt.Errorf("term range query must specify min or max") + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/query/wildcard.go b/vendor/github.com/blevesearch/bleve/search/query/wildcard.go new file mode 100644 index 00000000..7fd7482c --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/query/wildcard.go @@ -0,0 +1,106 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package query + +import ( + "regexp" + "strings" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/searcher" +) + +var wildcardRegexpReplacer = strings.NewReplacer( + // characters in the wildcard that must + // be escaped in the regexp + "+", `\+`, + "(", `\(`, + ")", `\)`, + "^", `\^`, + "$", `\$`, + ".", `\.`, + "{", `\{`, + "}", `\}`, + "[", `\[`, + "]", `\]`, + `|`, `\|`, + `\`, `\\`, + // wildcard characters + "*", ".*", + "?", ".") + +type WildcardQuery struct { + Wildcard string `json:"wildcard"` + FieldVal string `json:"field,omitempty"` + BoostVal *Boost `json:"boost,omitempty"` + compiled *regexp.Regexp +} + +// NewWildcardQuery creates a new Query which finds +// documents containing terms that match the +// specified wildcard. In the wildcard pattern '*' +// will match any sequence of 0 or more characters, +// and '?' will match any single character. +func NewWildcardQuery(wildcard string) *WildcardQuery { + return &WildcardQuery{ + Wildcard: wildcard, + } +} + +func (q *WildcardQuery) SetBoost(b float64) { + boost := Boost(b) + q.BoostVal = &boost +} + +func (q *WildcardQuery) Boost() float64 { + return q.BoostVal.Value() +} + +func (q *WildcardQuery) SetField(f string) { + q.FieldVal = f +} + +func (q *WildcardQuery) Field() string { + return q.FieldVal +} + +func (q *WildcardQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { + field := q.FieldVal + if q.FieldVal == "" { + field = m.DefaultSearchField() + } + if q.compiled == nil { + var err error + q.compiled, err = q.convertToRegexp() + if err != nil { + return nil, err + } + } + + return searcher.NewRegexpSearcher(i, q.compiled, field, q.BoostVal.Value(), options) +} + +func (q *WildcardQuery) Validate() error { + var err error + q.compiled, err = q.convertToRegexp() + return err +} + +func (q *WildcardQuery) convertToRegexp() (*regexp.Regexp, error) { + regexpString := wildcardRegexpReplacer.Replace(q.Wildcard) + return regexp.Compile(regexpString) +} diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go new file mode 100644 index 00000000..aad6f9c1 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "github.com/blevesearch/bleve/search" +) + +type ConjunctionQueryScorer struct { + options search.SearcherOptions +} + +func NewConjunctionQueryScorer(options search.SearcherOptions) *ConjunctionQueryScorer { + return &ConjunctionQueryScorer{ + options: options, + } +} + +func (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch) *search.DocumentMatch { + var sum float64 + var childrenExplanations []*search.Explanation + if s.options.Explain { + childrenExplanations = make([]*search.Explanation, len(constituents)) + } + + locations := []search.FieldTermLocationMap{} + for i, docMatch := range constituents { + sum += docMatch.Score + if s.options.Explain { + childrenExplanations[i] = docMatch.Expl + } + if docMatch.Locations != nil { + locations = append(locations, docMatch.Locations) + } + } + newScore := sum + var newExpl *search.Explanation + if s.options.Explain { + newExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} + } + + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl + if len(locations) == 1 { + rv.Locations = locations[0] + } else if len(locations) > 1 { + rv.Locations = search.MergeLocations(locations) + } + + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_constant.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go similarity index 61% rename from vendor/github.com/blevesearch/bleve/search/scorers/scorer_constant.go rename to vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go index 1434bd5e..a65a826f 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_constant.go +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go @@ -1,32 +1,38 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package scorers +package scorer import ( "fmt" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) type ConstantScorer struct { constant float64 boost float64 - explain bool + options search.SearcherOptions queryNorm float64 queryWeight float64 queryWeightExplanation *search.Explanation } -func NewConstantScorer(constant float64, boost float64, explain bool) *ConstantScorer { +func NewConstantScorer(constant float64, boost float64, options search.SearcherOptions) *ConstantScorer { rv := ConstantScorer{ - explain: explain, + options: options, queryWeight: 1.0, constant: constant, boost: boost, @@ -46,7 +52,7 @@ func (s *ConstantScorer) SetQueryNorm(qnorm float64) { // update the query weight s.queryWeight = s.boost * s.queryNorm - if s.explain { + if s.options.Explain { childrenExplanations := make([]*search.Explanation, 2) childrenExplanations[0] = &search.Explanation{ Value: s.boost, @@ -64,12 +70,12 @@ func (s *ConstantScorer) SetQueryNorm(qnorm float64) { } } -func (s *ConstantScorer) Score(id string) *search.DocumentMatch { +func (s *ConstantScorer) Score(ctx *search.SearchContext, id index.IndexInternalID) *search.DocumentMatch { var scoreExplanation *search.Explanation score := s.constant - if s.explain { + if s.options.Explain { scoreExplanation = &search.Explanation{ Value: score, Message: fmt.Sprintf("ConstantScore()"), @@ -79,7 +85,7 @@ func (s *ConstantScorer) Score(id string) *search.DocumentMatch { // if the query weight isn't 1, multiply if s.queryWeight != 1.0 { score = score * s.queryWeight - if s.explain { + if s.options.Explain { childExplanations := make([]*search.Explanation, 2) childExplanations[0] = s.queryWeightExplanation childExplanations[1] = scoreExplanation @@ -91,13 +97,12 @@ func (s *ConstantScorer) Score(id string) *search.DocumentMatch { } } - rv := search.DocumentMatch{ - ID: id, - Score: score, - } - if s.explain { + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = id + rv.Score = score + if s.options.Explain { rv.Expl = scoreExplanation } - return &rv + return rv } diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go new file mode 100644 index 00000000..184a15d2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go @@ -0,0 +1,77 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "fmt" + + "github.com/blevesearch/bleve/search" +) + +type DisjunctionQueryScorer struct { + options search.SearcherOptions +} + +func NewDisjunctionQueryScorer(options search.SearcherOptions) *DisjunctionQueryScorer { + return &DisjunctionQueryScorer{ + options: options, + } +} + +func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { + var sum float64 + var childrenExplanations []*search.Explanation + if s.options.Explain { + childrenExplanations = make([]*search.Explanation, len(constituents)) + } + + var locations []search.FieldTermLocationMap + for i, docMatch := range constituents { + sum += docMatch.Score + if s.options.Explain { + childrenExplanations[i] = docMatch.Expl + } + if docMatch.Locations != nil { + locations = append(locations, docMatch.Locations) + } + } + + var rawExpl *search.Explanation + if s.options.Explain { + rawExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} + } + + coord := float64(countMatch) / float64(countTotal) + newScore := sum * coord + var newExpl *search.Explanation + if s.options.Explain { + ce := make([]*search.Explanation, 2) + ce[0] = rawExpl + ce[1] = &search.Explanation{Value: coord, Message: fmt.Sprintf("coord(%d/%d)", countMatch, countTotal)} + newExpl = &search.Explanation{Value: newScore, Message: "product of:", Children: ce} + } + + // reuse constituents[0] as the return value + rv := constituents[0] + rv.Score = newScore + rv.Expl = newExpl + if len(locations) == 1 { + rv.Locations = locations[0] + } else if len(locations) > 1 { + rv.Locations = search.MergeLocations(locations) + } + + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_term.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go similarity index 64% rename from vendor/github.com/blevesearch/bleve/search/scorers/scorer_term.go rename to vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go index aefd69ea..b5f46322 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_term.go +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go @@ -1,13 +1,18 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package scorers +package scorer import ( "fmt" @@ -18,20 +23,20 @@ import ( ) type TermQueryScorer struct { - queryTerm string + queryTerm []byte queryField string queryBoost float64 docTerm uint64 docTotal uint64 idf float64 - explain bool + options search.SearcherOptions idfExplanation *search.Explanation queryNorm float64 queryWeight float64 queryWeightExplanation *search.Explanation } -func NewTermQueryScorer(queryTerm string, queryField string, queryBoost float64, docTotal, docTerm uint64, explain bool) *TermQueryScorer { +func NewTermQueryScorer(queryTerm []byte, queryField string, queryBoost float64, docTotal, docTerm uint64, options search.SearcherOptions) *TermQueryScorer { rv := TermQueryScorer{ queryTerm: queryTerm, queryField: queryField, @@ -39,11 +44,11 @@ func NewTermQueryScorer(queryTerm string, queryField string, queryBoost float64, docTerm: docTerm, docTotal: docTotal, idf: 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)), - explain: explain, + options: options, queryWeight: 1.0, } - if explain { + if options.Explain { rv.idfExplanation = &search.Explanation{ Value: rv.idf, Message: fmt.Sprintf("idf(docFreq=%d, maxDocs=%d)", docTerm, docTotal), @@ -64,7 +69,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { // update the query weight s.queryWeight = s.queryBoost * s.idf * s.queryNorm - if s.explain { + if s.options.Explain { childrenExplanations := make([]*search.Explanation, 3) childrenExplanations[0] = &search.Explanation{ Value: s.queryBoost, @@ -83,7 +88,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { } } -func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc) *search.DocumentMatch { +func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.TermFieldDoc) *search.DocumentMatch { var scoreExplanation *search.Explanation // need to compute score @@ -95,7 +100,7 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc) *search.DocumentM } score := tf * termMatch.Norm * s.idf - if s.explain { + if s.options.Explain { childrenExplanations := make([]*search.Explanation, 3) childrenExplanations[0] = &search.Explanation{ Value: tf, @@ -116,7 +121,7 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc) *search.DocumentM // if the query weight isn't 1, multiply if s.queryWeight != 1.0 { score = score * s.queryWeight - if s.explain { + if s.options.Explain { childExplanations := make([]*search.Explanation, 2) childExplanations[0] = s.queryWeightExplanation childExplanations[1] = scoreExplanation @@ -128,42 +133,50 @@ func (s *TermQueryScorer) Score(termMatch *index.TermFieldDoc) *search.DocumentM } } - rv := search.DocumentMatch{ - ID: termMatch.ID, - Score: score, - } - if s.explain { + rv := ctx.DocumentMatchPool.Get() + rv.IndexInternalID = append(rv.IndexInternalID, termMatch.ID...) + rv.Score = score + if s.options.Explain { rv.Expl = scoreExplanation } if termMatch.Vectors != nil && len(termMatch.Vectors) > 0 { + locs := make([]search.Location, len(termMatch.Vectors)) + locsUsed := 0 + + totalPositions := 0 + for _, v := range termMatch.Vectors { + totalPositions += len(v.ArrayPositions) + } + positions := make(search.ArrayPositions, totalPositions) + positionsUsed := 0 rv.Locations = make(search.FieldTermLocationMap) for _, v := range termMatch.Vectors { tlm := rv.Locations[v.Field] if tlm == nil { tlm = make(search.TermLocationMap) + rv.Locations[v.Field] = tlm } - loc := search.Location{ - Pos: float64(v.Pos), - Start: float64(v.Start), - End: float64(v.End), + loc := &locs[locsUsed] + locsUsed++ + + loc.Pos = v.Pos + loc.Start = v.Start + loc.End = v.End + + if len(v.ArrayPositions) > 0 { + loc.ArrayPositions = positions[positionsUsed : positionsUsed+len(v.ArrayPositions)] + for i, ap := range v.ArrayPositions { + loc.ArrayPositions[i] = ap + } + positionsUsed += len(v.ArrayPositions) } - locations := tlm[s.queryTerm] - if locations == nil { - locations = make(search.Locations, 1) - locations[0] = &loc - } else { - locations = append(locations, &loc) - } - tlm[s.queryTerm] = locations - - rv.Locations[v.Field] = tlm + tlm[string(s.queryTerm)] = append(tlm[string(s.queryTerm)], loc) } - } - return &rv + return rv } diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go b/vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go new file mode 100644 index 00000000..e26d33d9 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/scorer/sqrt_cache.go @@ -0,0 +1,30 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorer + +import ( + "math" +) + +var SqrtCache []float64 + +const MaxSqrtCache = 64 + +func init() { + SqrtCache = make([]float64, MaxSqrtCache) + for i := 0; i < MaxSqrtCache; i++ { + SqrtCache[i] = math.Sqrt(float64(i)) + } +} diff --git a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_conjunction.go b/vendor/github.com/blevesearch/bleve/search/scorers/scorer_conjunction.go deleted file mode 100644 index 422f282c..00000000 --- a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_conjunction.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package scorers - -import ( - "github.com/blevesearch/bleve/search" -) - -type ConjunctionQueryScorer struct { - explain bool -} - -func NewConjunctionQueryScorer(explain bool) *ConjunctionQueryScorer { - return &ConjunctionQueryScorer{ - explain: explain, - } -} - -func (s *ConjunctionQueryScorer) Score(constituents []*search.DocumentMatch) *search.DocumentMatch { - rv := search.DocumentMatch{ - ID: constituents[0].ID, - } - - var sum float64 - var childrenExplanations []*search.Explanation - if s.explain { - childrenExplanations = make([]*search.Explanation, len(constituents)) - } - - locations := []search.FieldTermLocationMap{} - for i, docMatch := range constituents { - sum += docMatch.Score - if s.explain { - childrenExplanations[i] = docMatch.Expl - } - if docMatch.Locations != nil { - locations = append(locations, docMatch.Locations) - } - } - rv.Score = sum - if s.explain { - rv.Expl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} - } - - if len(locations) == 1 { - rv.Locations = locations[0] - } else if len(locations) > 1 { - rv.Locations = search.MergeLocations(locations) - } - - return &rv -} diff --git a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_constant_test.go b/vendor/github.com/blevesearch/bleve/search/scorers/scorer_constant_test.go deleted file mode 100644 index e7f220fd..00000000 --- a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_constant_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) 2013 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package scorers - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -func TestConstantScorer(t *testing.T) { - - scorer := NewConstantScorer(1, 1, true) - - tests := []struct { - termMatch *index.TermFieldDoc - result *search.DocumentMatch - }{ - // test some simple math - { - termMatch: &index.TermFieldDoc{ - ID: "one", - Freq: 1, - Norm: 1.0, - Vectors: []*index.TermFieldVector{ - &index.TermFieldVector{ - Field: "desc", - Pos: 1, - Start: 0, - End: 4, - }, - }, - }, - result: &search.DocumentMatch{ - ID: "one", - Score: 1.0, - Expl: &search.Explanation{ - Value: 1.0, - Message: "ConstantScore()", - }, - }, - }, - } - - for _, test := range tests { - actual := scorer.Score(test.termMatch.ID) - - if !reflect.DeepEqual(actual, test.result) { - t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) - } - } - -} - -func TestConstantScorerWithQueryNorm(t *testing.T) { - - scorer := NewConstantScorer(1, 1, true) - scorer.SetQueryNorm(2.0) - - tests := []struct { - termMatch *index.TermFieldDoc - result *search.DocumentMatch - }{ - { - termMatch: &index.TermFieldDoc{ - ID: "one", - Freq: 1, - Norm: 1.0, - }, - result: &search.DocumentMatch{ - ID: "one", - Score: 2.0, - Expl: &search.Explanation{ - Value: 2.0, - Message: "weight(^1.000000), product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: 2.0, - Message: "ConstantScore()^1.000000, product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: 1, - Message: "boost", - }, - &search.Explanation{ - Value: 2, - Message: "queryNorm", - }, - }, - }, - &search.Explanation{ - Value: 1.0, - Message: "ConstantScore()", - }, - }, - }, - }, - }, - } - - for _, test := range tests { - actual := scorer.Score(test.termMatch.ID) - - if !reflect.DeepEqual(actual, test.result) { - t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) - } - } - -} diff --git a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_disjunction.go b/vendor/github.com/blevesearch/bleve/search/scorers/scorer_disjunction.go deleted file mode 100644 index 00bc8cd0..00000000 --- a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_disjunction.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package scorers - -import ( - "fmt" - - "github.com/blevesearch/bleve/search" -) - -type DisjunctionQueryScorer struct { - explain bool -} - -func NewDisjunctionQueryScorer(explain bool) *DisjunctionQueryScorer { - return &DisjunctionQueryScorer{ - explain: explain, - } -} - -func (s *DisjunctionQueryScorer) Score(constituents []*search.DocumentMatch, countMatch, countTotal int) *search.DocumentMatch { - rv := search.DocumentMatch{ - ID: constituents[0].ID, - } - - var sum float64 - var childrenExplanations []*search.Explanation - if s.explain { - childrenExplanations = make([]*search.Explanation, len(constituents)) - } - - locations := []search.FieldTermLocationMap{} - for i, docMatch := range constituents { - sum += docMatch.Score - if s.explain { - childrenExplanations[i] = docMatch.Expl - } - if docMatch.Locations != nil { - locations = append(locations, docMatch.Locations) - } - } - - var rawExpl *search.Explanation - if s.explain { - rawExpl = &search.Explanation{Value: sum, Message: "sum of:", Children: childrenExplanations} - } - - coord := float64(countMatch) / float64(countTotal) - rv.Score = sum * coord - if s.explain { - ce := make([]*search.Explanation, 2) - ce[0] = rawExpl - ce[1] = &search.Explanation{Value: coord, Message: fmt.Sprintf("coord(%d/%d)", countMatch, countTotal)} - rv.Expl = &search.Explanation{Value: rv.Score, Message: "product of:", Children: ce} - } - - if len(locations) == 1 { - rv.Locations = locations[0] - } else if len(locations) > 1 { - rv.Locations = search.MergeLocations(locations) - } - - return &rv -} diff --git a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_term_test.go b/vendor/github.com/blevesearch/bleve/search/scorers/scorer_term_test.go deleted file mode 100644 index 59a08f1b..00000000 --- a/vendor/github.com/blevesearch/bleve/search/scorers/scorer_term_test.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) 2013 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package scorers - -import ( - "math" - "reflect" - "testing" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -func TestTermScorer(t *testing.T) { - - var docTotal uint64 = 100 - var docTerm uint64 = 9 - var queryTerm = "beer" - var queryField = "desc" - var queryBoost = 1.0 - scorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, true) - idf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)) - - tests := []struct { - termMatch *index.TermFieldDoc - result *search.DocumentMatch - }{ - // test some simple math - { - termMatch: &index.TermFieldDoc{ - ID: "one", - Freq: 1, - Norm: 1.0, - Vectors: []*index.TermFieldVector{ - &index.TermFieldVector{ - Field: "desc", - Pos: 1, - Start: 0, - End: 4, - }, - }, - }, - result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(1.0) * idf, - Expl: &search.Explanation{ - Value: math.Sqrt(1.0) * idf, - Message: "fieldWeight(desc:beer in one), product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: 1, - Message: "tf(termFreq(desc:beer)=1", - }, - &search.Explanation{ - Value: 1, - Message: "fieldNorm(field=desc, doc=one)", - }, - &search.Explanation{ - Value: idf, - Message: "idf(docFreq=9, maxDocs=100)", - }, - }, - }, - Locations: search.FieldTermLocationMap{ - "desc": search.TermLocationMap{ - "beer": search.Locations{ - &search.Location{ - Pos: 1, - Start: 0, - End: 4, - }, - }, - }, - }, - }, - }, - // test the same thing again (score should be cached this time) - { - termMatch: &index.TermFieldDoc{ - ID: "one", - Freq: 1, - Norm: 1.0, - }, - result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(1.0) * idf, - Expl: &search.Explanation{ - Value: math.Sqrt(1.0) * idf, - Message: "fieldWeight(desc:beer in one), product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: 1, - Message: "tf(termFreq(desc:beer)=1", - }, - &search.Explanation{ - Value: 1, - Message: "fieldNorm(field=desc, doc=one)", - }, - &search.Explanation{ - Value: idf, - Message: "idf(docFreq=9, maxDocs=100)", - }, - }, - }, - }, - }, - // test a case where the sqrt isn't precalculated - { - termMatch: &index.TermFieldDoc{ - ID: "one", - Freq: 65, - Norm: 1.0, - }, - result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(65) * idf, - Expl: &search.Explanation{ - Value: math.Sqrt(65) * idf, - Message: "fieldWeight(desc:beer in one), product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: math.Sqrt(65), - Message: "tf(termFreq(desc:beer)=65", - }, - &search.Explanation{ - Value: 1, - Message: "fieldNorm(field=desc, doc=one)", - }, - &search.Explanation{ - Value: idf, - Message: "idf(docFreq=9, maxDocs=100)", - }, - }, - }, - }, - }, - } - - for _, test := range tests { - actual := scorer.Score(test.termMatch) - - if !reflect.DeepEqual(actual, test.result) { - t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) - } - } - -} - -func TestTermScorerWithQueryNorm(t *testing.T) { - - var docTotal uint64 = 100 - var docTerm uint64 = 9 - var queryTerm = "beer" - var queryField = "desc" - var queryBoost = 3.0 - scorer := NewTermQueryScorer(queryTerm, queryField, queryBoost, docTotal, docTerm, true) - idf := 1.0 + math.Log(float64(docTotal)/float64(docTerm+1.0)) - - scorer.SetQueryNorm(2.0) - - expectedQueryWeight := 3 * idf * 3 * idf - actualQueryWeight := scorer.Weight() - if expectedQueryWeight != actualQueryWeight { - t.Errorf("expected query weight %f, got %f", expectedQueryWeight, actualQueryWeight) - } - - tests := []struct { - termMatch *index.TermFieldDoc - result *search.DocumentMatch - }{ - { - termMatch: &index.TermFieldDoc{ - ID: "one", - Freq: 1, - Norm: 1.0, - }, - result: &search.DocumentMatch{ - ID: "one", - Score: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, - Expl: &search.Explanation{ - Value: math.Sqrt(1.0) * idf * 3.0 * idf * 2.0, - Message: "weight(desc:beer^3.000000 in one), product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: 2.0 * idf * 3.0, - Message: "queryWeight(desc:beer^3.000000), product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: 3, - Message: "boost", - }, - &search.Explanation{ - Value: idf, - Message: "idf(docFreq=9, maxDocs=100)", - }, - &search.Explanation{ - Value: 2, - Message: "queryNorm", - }, - }, - }, - &search.Explanation{ - Value: math.Sqrt(1.0) * idf, - Message: "fieldWeight(desc:beer in one), product of:", - Children: []*search.Explanation{ - &search.Explanation{ - Value: 1, - Message: "tf(termFreq(desc:beer)=1", - }, - &search.Explanation{ - Value: 1, - Message: "fieldNorm(field=desc, doc=one)", - }, - &search.Explanation{ - Value: idf, - Message: "idf(docFreq=9, maxDocs=100)", - }, - }, - }, - }, - }, - }, - }, - } - - for _, test := range tests { - actual := scorer.Score(test.termMatch) - - if !reflect.DeepEqual(actual, test.result) { - t.Errorf("expected %#v got %#v for %#v", test.result, actual, test.termMatch) - } - } - -} diff --git a/vendor/github.com/blevesearch/bleve/search/scorers/sqrt_cache.go b/vendor/github.com/blevesearch/bleve/search/scorers/sqrt_cache.go deleted file mode 100644 index f93d27cc..00000000 --- a/vendor/github.com/blevesearch/bleve/search/scorers/sqrt_cache.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package scorers - -import ( - "math" -) - -var SqrtCache map[int]float64 - -const MaxSqrtCache = 64 - -func init() { - SqrtCache = make(map[int]float64, MaxSqrtCache) - for i := 0; i < MaxSqrtCache; i++ { - SqrtCache[i] = math.Sqrt(float64(i)) - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/search.go b/vendor/github.com/blevesearch/bleve/search/search.go index 8eb473dd..a9bf9db9 100644 --- a/vendor/github.com/blevesearch/bleve/search/search.go +++ b/vendor/github.com/blevesearch/bleve/search/search.go @@ -1,18 +1,45 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package search +import ( + "fmt" + + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" +) + +type ArrayPositions []uint64 + +func (ap ArrayPositions) Equals(other ArrayPositions) bool { + if len(ap) != len(other) { + return false + } + for i := range ap { + if ap[i] != other[i] { + return false + } + } + return true +} + type Location struct { - Pos float64 `json:"pos"` - Start float64 `json:"start"` - End float64 `json:"end"` + Pos uint64 `json:"pos"` + Start uint64 `json:"start"` + End uint64 `json:"end"` + ArrayPositions ArrayPositions `json:"array_positions"` } type Locations []*Location @@ -20,15 +47,7 @@ type Locations []*Location type TermLocationMap map[string]Locations func (t TermLocationMap) AddLocation(term string, location *Location) { - existingLocations, exists := t[term] - if exists { - existingLocations = append(existingLocations, location) - t[term] = existingLocations - } else { - locations := make(Locations, 1) - locations[0] = location - t[term] = locations - } + t[term] = append(t[term], location) } type FieldTermLocationMap map[string]TermLocationMap @@ -36,12 +55,25 @@ type FieldTermLocationMap map[string]TermLocationMap type FieldFragmentMap map[string][]string type DocumentMatch struct { - ID string `json:"id"` - Score float64 `json:"score"` - Expl *Explanation `json:"explanation,omitempty"` - Locations FieldTermLocationMap `json:"locations,omitempty"` - Fragments FieldFragmentMap `json:"fragments,omitempty"` - Fields map[string]interface{} `json:"fields,omitempty"` + Index string `json:"index,omitempty"` + ID string `json:"id"` + IndexInternalID index.IndexInternalID `json:"-"` + Score float64 `json:"score"` + Expl *Explanation `json:"explanation,omitempty"` + Locations FieldTermLocationMap `json:"locations,omitempty"` + Fragments FieldFragmentMap `json:"fragments,omitempty"` + Sort []string `json:"sort,omitempty"` + + // Fields contains the values for document fields listed in + // SearchRequest.Fields. Text fields are returned as strings, numeric + // fields as float64s and date fields as time.RFC3339 formatted strings. + Fields map[string]interface{} `json:"fields,omitempty"` + + // if we load the document for this hit, remember it so we dont load again + Document *document.Document `json:"-"` + + // used to maintain natural index order + HitNumber uint64 `json:"-"` } func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { @@ -49,19 +81,39 @@ func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { dm.Fields = make(map[string]interface{}) } existingVal, ok := dm.Fields[name] - if ok { - valSlice, ok := existingVal.([]interface{}) - if ok { - // already a slice, append to it - valSlice = append(valSlice, value) - } else { - // create a slice - valSlice = []interface{}{existingVal, value} - } - dm.Fields[name] = valSlice - } else { + if !ok { dm.Fields[name] = value + return } + + valSlice, ok := existingVal.([]interface{}) + if ok { + // already a slice, append to it + valSlice = append(valSlice, value) + } else { + // create a slice + valSlice = []interface{}{existingVal, value} + } + dm.Fields[name] = valSlice +} + +// Reset allows an already allocated DocumentMatch to be reused +func (dm *DocumentMatch) Reset() *DocumentMatch { + // remember the []byte used for the IndexInternalID + indexInternalID := dm.IndexInternalID + // remember the []interface{} used for sort + sort := dm.Sort + // idiom to copy over from empty DocumentMatch (0 allocations) + *dm = DocumentMatch{} + // reuse the []byte already allocated (and reset len to 0) + dm.IndexInternalID = indexInternalID[:0] + // reuse the []interface{} already allocated (and reset len to 0) + dm.Sort = sort[:0] + return dm +} + +func (dm *DocumentMatch) String() string { + return fmt.Sprintf("[%s-%f]", string(dm.IndexInternalID), dm.Score) } type DocumentMatchCollection []*DocumentMatch @@ -71,11 +123,23 @@ func (c DocumentMatchCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } func (c DocumentMatchCollection) Less(i, j int) bool { return c[i].Score > c[j].Score } type Searcher interface { - Next() (*DocumentMatch, error) - Advance(ID string) (*DocumentMatch, error) + Next(ctx *SearchContext) (*DocumentMatch, error) + Advance(ctx *SearchContext, ID index.IndexInternalID) (*DocumentMatch, error) Close() error Weight() float64 SetQueryNorm(float64) Count() uint64 Min() int + + DocumentMatchPoolSize() int +} + +type SearcherOptions struct { + Explain bool + IncludeTermVectors bool +} + +// SearchContext represents the context around a single search +type SearchContext struct { + DocumentMatchPool *DocumentMatchPool } diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go b/vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go new file mode 100644 index 00000000..536c593c --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/ordered_searchers_list.go @@ -0,0 +1,35 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/search" +) + +type OrderedSearcherList []search.Searcher + +// sort.Interface + +func (otrl OrderedSearcherList) Len() int { + return len(otrl) +} + +func (otrl OrderedSearcherList) Less(i, j int) bool { + return otrl[i].Count() < otrl[j].Count() +} + +func (otrl OrderedSearcherList) Swap(i, j int) { + otrl[i], otrl[j] = otrl[j], otrl[i] +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go new file mode 100644 index 00000000..a905c29e --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go @@ -0,0 +1,391 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "math" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" +) + +type BooleanSearcher struct { + indexReader index.IndexReader + mustSearcher search.Searcher + shouldSearcher search.Searcher + mustNotSearcher search.Searcher + queryNorm float64 + currMust *search.DocumentMatch + currShould *search.DocumentMatch + currMustNot *search.DocumentMatch + currentID index.IndexInternalID + min uint64 + scorer *scorer.ConjunctionQueryScorer + matches []*search.DocumentMatch + initialized bool +} + +func NewBooleanSearcher(indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) { + // build our searcher + rv := BooleanSearcher{ + indexReader: indexReader, + mustSearcher: mustSearcher, + shouldSearcher: shouldSearcher, + mustNotSearcher: mustNotSearcher, + scorer: scorer.NewConjunctionQueryScorer(options), + matches: make([]*search.DocumentMatch, 2), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *BooleanSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + if s.mustSearcher != nil { + sumOfSquaredWeights += s.mustSearcher.Weight() + } + if s.shouldSearcher != nil { + sumOfSquaredWeights += s.shouldSearcher.Weight() + } + + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + if s.mustSearcher != nil { + s.mustSearcher.SetQueryNorm(s.queryNorm) + } + if s.shouldSearcher != nil { + s.shouldSearcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *BooleanSearcher) initSearchers(ctx *search.SearchContext) error { + var err error + // get all searchers pointing at their first match + if s.mustSearcher != nil { + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.shouldSearcher != nil { + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.mustNotSearcher != nil { + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.mustSearcher != nil && s.currMust != nil { + s.currentID = s.currMust.IndexInternalID + } else if s.mustSearcher == nil && s.currShould != nil { + s.currentID = s.currShould.IndexInternalID + } else { + s.currentID = nil + } + + s.initialized = true + return nil +} + +func (s *BooleanSearcher) advanceNextMust(ctx *search.SearchContext, skipReturn *search.DocumentMatch) error { + var err error + + if s.mustSearcher != nil { + if s.currMust != skipReturn { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Next(ctx) + if err != nil { + return err + } + } else { + if s.currShould != skipReturn { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Next(ctx) + if err != nil { + return err + } + } + + if s.mustSearcher != nil && s.currMust != nil { + s.currentID = s.currMust.IndexInternalID + } else if s.mustSearcher == nil && s.currShould != nil { + s.currentID = s.currShould.IndexInternalID + } else { + s.currentID = nil + } + return nil +} + +func (s *BooleanSearcher) Weight() float64 { + var rv float64 + if s.mustSearcher != nil { + rv += s.mustSearcher.Weight() + } + if s.shouldSearcher != nil { + rv += s.shouldSearcher.Weight() + } + + return rv +} + +func (s *BooleanSearcher) SetQueryNorm(qnorm float64) { + if s.mustSearcher != nil { + s.mustSearcher.SetQueryNorm(qnorm) + } + if s.shouldSearcher != nil { + s.shouldSearcher.SetQueryNorm(qnorm) + } +} + +func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + var err error + var rv *search.DocumentMatch + + for s.currentID != nil { + if s.currMustNot != nil { + cmp := s.currMustNot.IndexInternalID.Compare(s.currentID) + if cmp < 0 { + ctx.DocumentMatchPool.Put(s.currMustNot) + // advance must not searcher to our candidate entry + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, s.currentID) + if err != nil { + return nil, err + } + if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) { + // the candidate is excluded + err = s.advanceNextMust(ctx, nil) + if err != nil { + return nil, err + } + continue + } + } else if cmp == 0 { + // the candidate is excluded + err = s.advanceNextMust(ctx, nil) + if err != nil { + return nil, err + } + continue + } + } + + shouldCmpOrNil := 1 // NOTE: shouldCmp will also be 1 when currShould == nil. + if s.currShould != nil { + shouldCmpOrNil = s.currShould.IndexInternalID.Compare(s.currentID) + } + + if shouldCmpOrNil < 0 { + ctx.DocumentMatchPool.Put(s.currShould) + // advance should searcher to our candidate entry + s.currShould, err = s.shouldSearcher.Advance(ctx, s.currentID) + if err != nil { + return nil, err + } + if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) { + // score bonus matches should + var cons []*search.DocumentMatch + if s.currMust != nil { + cons = s.matches + cons[0] = s.currMust + cons[1] = s.currShould + } else { + cons = s.matches[0:1] + cons[0] = s.currShould + } + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } else if s.shouldSearcher.Min() == 0 { + // match is OK anyway + cons := s.matches[0:1] + cons[0] = s.currMust + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } + } else if shouldCmpOrNil == 0 { + // score bonus matches should + var cons []*search.DocumentMatch + if s.currMust != nil { + cons = s.matches + cons[0] = s.currMust + cons[1] = s.currShould + } else { + cons = s.matches[0:1] + cons[0] = s.currShould + } + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 { + // match is OK anyway + cons := s.matches[0:1] + cons[0] = s.currMust + rv = s.scorer.Score(ctx, cons) + err = s.advanceNextMust(ctx, rv) + if err != nil { + return nil, err + } + break + } + + err = s.advanceNextMust(ctx, nil) + if err != nil { + return nil, err + } + } + return rv, nil +} + +func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + var err error + if s.mustSearcher != nil { + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } + s.currMust, err = s.mustSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + if s.shouldSearcher != nil { + if s.currShould != nil { + ctx.DocumentMatchPool.Put(s.currShould) + } + s.currShould, err = s.shouldSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + if s.mustNotSearcher != nil { + if s.currMustNot != nil { + ctx.DocumentMatchPool.Put(s.currMustNot) + } + s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + + if s.mustSearcher != nil && s.currMust != nil { + s.currentID = s.currMust.IndexInternalID + } else if s.mustSearcher == nil && s.currShould != nil { + s.currentID = s.currShould.IndexInternalID + } else { + s.currentID = nil + } + + return s.Next(ctx) +} + +func (s *BooleanSearcher) Count() uint64 { + + // for now return a worst case + var sum uint64 + if s.mustSearcher != nil { + sum += s.mustSearcher.Count() + } + if s.shouldSearcher != nil { + sum += s.shouldSearcher.Count() + } + return sum +} + +func (s *BooleanSearcher) Close() error { + var err0, err1, err2 error + if s.mustSearcher != nil { + err0 = s.mustSearcher.Close() + } + if s.shouldSearcher != nil { + err1 = s.shouldSearcher.Close() + } + if s.mustNotSearcher != nil { + err2 = s.mustNotSearcher.Close() + } + if err0 != nil { + return err0 + } + if err1 != nil { + return err1 + } + if err2 != nil { + return err2 + } + return nil +} + +func (s *BooleanSearcher) Min() int { + return 0 +} + +func (s *BooleanSearcher) DocumentMatchPoolSize() int { + rv := 3 + if s.mustSearcher != nil { + rv += s.mustSearcher.DocumentMatchPoolSize() + } + if s.shouldSearcher != nil { + rv += s.shouldSearcher.DocumentMatchPoolSize() + } + if s.mustNotSearcher != nil { + rv += s.mustNotSearcher.DocumentMatchPoolSize() + } + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go new file mode 100644 index 00000000..9ab0e7fa --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go @@ -0,0 +1,232 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "math" + "sort" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" +) + +type ConjunctionSearcher struct { + indexReader index.IndexReader + searchers OrderedSearcherList + queryNorm float64 + currs []*search.DocumentMatch + maxIDIdx int + scorer *scorer.ConjunctionQueryScorer + initialized bool + options search.SearcherOptions +} + +func NewConjunctionSearcher(indexReader index.IndexReader, qsearchers []search.Searcher, options search.SearcherOptions) (*ConjunctionSearcher, error) { + // build the downstream searchers + searchers := make(OrderedSearcherList, len(qsearchers)) + for i, searcher := range qsearchers { + searchers[i] = searcher + } + // sort the searchers + sort.Sort(searchers) + // build our searcher + rv := ConjunctionSearcher{ + indexReader: indexReader, + options: options, + searchers: searchers, + currs: make([]*search.DocumentMatch, len(searchers)), + scorer: scorer.NewConjunctionQueryScorer(options), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *ConjunctionSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, termSearcher := range s.searchers { + sumOfSquaredWeights += termSearcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, termSearcher := range s.searchers { + termSearcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error { + var err error + // get all searchers pointing at their first match + for i, termSearcher := range s.searchers { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) + if err != nil { + return err + } + } + s.initialized = true + return nil +} + +func (s *ConjunctionSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *ConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + var rv *search.DocumentMatch + var err error +OUTER: + for s.currs[s.maxIDIdx] != nil { + maxID := s.currs[s.maxIDIdx].IndexInternalID + + i := 0 + for i < len(s.currs) { + if s.currs[i] == nil { + return nil, nil + } + + if i == s.maxIDIdx { + i++ + continue + } + + cmp := maxID.Compare(s.currs[i].IndexInternalID) + if cmp == 0 { + i++ + continue + } + + if cmp < 0 { + // maxID < currs[i], so we found a new maxIDIdx + s.maxIDIdx = i + + // advance the positions where [0 <= x < i], since we + // know they were equal to the former max entry + maxID = s.currs[s.maxIDIdx].IndexInternalID + for x := 0; x < i; x++ { + err = s.advanceChild(ctx, x, maxID) + if err != nil { + return nil, err + } + } + + continue OUTER + } + + // maxID > currs[i], so need to advance searchers[i] + err = s.advanceChild(ctx, i, maxID) + if err != nil { + return nil, err + } + + // don't bump i, so that we'll examine the just-advanced + // currs[i] again + } + + // if we get here, a doc matched all readers, so score and add it + rv = s.scorer.Score(ctx, s.currs) + + // we know all the searchers are pointing at the same thing + // so they all need to be bumped + for i, termSearcher := range s.searchers { + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) + if err != nil { + return nil, err + } + } + + // don't continue now, wait for the next call to Next() + break + } + return rv, nil +} + +func (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + for i := range s.searchers { + err := s.advanceChild(ctx, i, ID) + if err != nil { + return nil, err + } + } + return s.Next(ctx) +} + +func (s *ConjunctionSearcher) advanceChild(ctx *search.SearchContext, i int, ID index.IndexInternalID) (err error) { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = s.searchers[i].Advance(ctx, ID) + return err +} + +func (s *ConjunctionSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *ConjunctionSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *ConjunctionSearcher) Min() int { + return 0 +} + +func (s *ConjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go new file mode 100644 index 00000000..96bd5447 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go @@ -0,0 +1,271 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "fmt" + "math" + "sort" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" +) + +// DisjunctionMaxClauseCount is a compile time setting that applications can +// adjust to non-zero value to cause the DisjunctionSearcher to return an +// error instead of exeucting searches when the size exceeds this value. +var DisjunctionMaxClauseCount = 0 + +type DisjunctionSearcher struct { + indexReader index.IndexReader + searchers OrderedSearcherList + numSearchers int + queryNorm float64 + currs []*search.DocumentMatch + scorer *scorer.DisjunctionQueryScorer + min int + matching []*search.DocumentMatch + matchingIdxs []int + initialized bool +} + +func tooManyClauses(count int) bool { + if DisjunctionMaxClauseCount != 0 && count > DisjunctionMaxClauseCount { + return true + } + return false +} + +func tooManyClausesErr() error { + return fmt.Errorf("TooManyClauses[maxClauseCount is set to %d]", + DisjunctionMaxClauseCount) +} + +func NewDisjunctionSearcher(indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions) ( + *DisjunctionSearcher, error) { + return newDisjunctionSearcher(indexReader, qsearchers, min, options, + true) +} + +func newDisjunctionSearcher(indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) ( + *DisjunctionSearcher, error) { + if limit && tooManyClauses(len(qsearchers)) { + return nil, tooManyClausesErr() + } + // build the downstream searchers + searchers := make(OrderedSearcherList, len(qsearchers)) + for i, searcher := range qsearchers { + searchers[i] = searcher + } + // sort the searchers + sort.Sort(sort.Reverse(searchers)) + // build our searcher + rv := DisjunctionSearcher{ + indexReader: indexReader, + searchers: searchers, + numSearchers: len(searchers), + currs: make([]*search.DocumentMatch, len(searchers)), + scorer: scorer.NewDisjunctionQueryScorer(options), + min: int(min), + matching: make([]*search.DocumentMatch, len(searchers)), + matchingIdxs: make([]int, len(searchers)), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *DisjunctionSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, termSearcher := range s.searchers { + sumOfSquaredWeights += termSearcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, termSearcher := range s.searchers { + termSearcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *DisjunctionSearcher) initSearchers(ctx *search.SearchContext) error { + var err error + // get all searchers pointing at their first match + for i, termSearcher := range s.searchers { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Next(ctx) + if err != nil { + return err + } + } + + err = s.updateMatches() + if err != nil { + return err + } + + s.initialized = true + return nil +} + +func (s *DisjunctionSearcher) updateMatches() error { + matching := s.matching[:0] + matchingIdxs := s.matchingIdxs[:0] + + for i := 0; i < len(s.currs); i++ { + curr := s.currs[i] + if curr == nil { + continue + } + + if len(matching) > 0 { + cmp := curr.IndexInternalID.Compare(matching[0].IndexInternalID) + if cmp > 0 { + continue + } + + if cmp < 0 { + matching = matching[:0] + matchingIdxs = matchingIdxs[:0] + } + } + + matching = append(matching, curr) + matchingIdxs = append(matchingIdxs, i) + } + + s.matching = matching + s.matchingIdxs = matchingIdxs + + return nil +} + +func (s *DisjunctionSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) ( + *search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + var err error + var rv *search.DocumentMatch + + found := false + for !found && len(s.matching) > 0 { + if len(s.matching) >= s.min { + found = true + // score this match + rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + } + + // invoke next on all the matching searchers + for _, i := range s.matchingIdxs { + searcher := s.searchers[i] + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return nil, err + } + } + + err = s.updateMatches() + if err != nil { + return nil, err + } + } + return rv, nil +} + +func (s *DisjunctionSearcher) Advance(ctx *search.SearchContext, + ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + // get all searchers pointing at their first match + var err error + for i, termSearcher := range s.searchers { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = termSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + + err = s.updateMatches() + if err != nil { + return nil, err + } + + return s.Next(ctx) +} + +func (s *DisjunctionSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *DisjunctionSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *DisjunctionSearcher) Min() int { + return s.min +} + +func (s *DisjunctionSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go new file mode 100644 index 00000000..06351b4a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go @@ -0,0 +1,93 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" +) + +// DocIDSearcher returns documents matching a predefined set of identifiers. +type DocIDSearcher struct { + reader index.DocIDReader + scorer *scorer.ConstantScorer + count int +} + +func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64, + options search.SearcherOptions) (searcher *DocIDSearcher, err error) { + + reader, err := indexReader.DocIDReaderOnly(ids) + if err != nil { + return nil, err + } + scorer := scorer.NewConstantScorer(1.0, boost, options) + return &DocIDSearcher{ + scorer: scorer, + reader: reader, + count: len(ids), + }, nil +} + +func (s *DocIDSearcher) Count() uint64 { + return uint64(s.count) +} + +func (s *DocIDSearcher) Weight() float64 { + return s.scorer.Weight() +} + +func (s *DocIDSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *DocIDSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + docidMatch, err := s.reader.Next() + if err != nil { + return nil, err + } + if docidMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(ctx, docidMatch) + return docMatch, nil +} + +func (s *DocIDSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + docidMatch, err := s.reader.Advance(ID) + if err != nil { + return nil, err + } + if docidMatch == nil { + return nil, nil + } + + docMatch := s.scorer.Score(ctx, docidMatch) + return docMatch, nil +} + +func (s *DocIDSearcher) Close() error { + return s.reader.Close() +} + +func (s *DocIDSearcher) Min() int { + return 0 +} + +func (s *DocIDSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go new file mode 100644 index 00000000..219f2ee7 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go @@ -0,0 +1,88 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +// FilterFunc defines a function which can filter documents +// returning true means keep the document +// returning false means do not keep the document +type FilterFunc func(d *search.DocumentMatch) bool + +// FilteringSearcher wraps any other searcher, but checks any Next/Advance +// call against the supplied FilterFunc +type FilteringSearcher struct { + child search.Searcher + accept FilterFunc +} + +func NewFilteringSearcher(s search.Searcher, filter FilterFunc) *FilteringSearcher { + return &FilteringSearcher{ + child: s, + accept: filter, + } +} + +func (f *FilteringSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + next, err := f.child.Next(ctx) + for next != nil && err == nil { + if f.accept(next) { + return next, nil + } + next, err = f.child.Next(ctx) + } + return nil, err +} + +func (f *FilteringSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + adv, err := f.child.Advance(ctx, ID) + if err != nil { + return nil, err + } + if adv == nil { + return nil, nil + } + if f.accept(adv) { + return adv, nil + } + return f.Next(ctx) +} + +func (f *FilteringSearcher) Close() error { + return f.child.Close() +} + +func (f *FilteringSearcher) Weight() float64 { + return f.child.Weight() +} + +func (f *FilteringSearcher) SetQueryNorm(n float64) { + f.child.SetQueryNorm(n) +} + +func (f *FilteringSearcher) Count() uint64 { + return f.child.Count() +} + +func (f *FilteringSearcher) Min() int { + return f.child.Min() +} + +func (f *FilteringSearcher) DocumentMatchPoolSize() int { + return f.child.DocumentMatchPoolSize() +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go new file mode 100644 index 00000000..90abaa0a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go @@ -0,0 +1,74 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +func NewFuzzySearcher(indexReader index.IndexReader, term string, + prefix, fuzziness int, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + // Note: we don't byte slice the term for a prefix because of runes. + prefixTerm := "" + for i, r := range term { + if i < prefix { + prefixTerm += string(r) + } else { + break + } + } + + candidateTerms, err := findFuzzyCandidateTerms(indexReader, term, fuzziness, + field, prefixTerm) + if err != nil { + return nil, err + } + + return NewMultiTermSearcher(indexReader, candidateTerms, field, + boost, options, true) +} + +func findFuzzyCandidateTerms(indexReader index.IndexReader, term string, + fuzziness int, field, prefixTerm string) (rv []string, err error) { + rv = make([]string, 0) + var fieldDict index.FieldDict + if len(prefixTerm) > 0 { + fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) + } else { + fieldDict, err = indexReader.FieldDict(field) + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // enumerate terms and check levenshtein distance + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + ld, exceeded := search.LevenshteinDistanceMax(term, tfd.Term, fuzziness) + if !exceeded && ld <= fuzziness { + rv = append(rv, tfd.Term) + if tooManyClauses(len(rv)) { + return rv, tooManyClausesErr() + } + } + tfd, err = fieldDict.Next() + } + + return rv, err +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go new file mode 100644 index 00000000..f8b1b4cf --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_geoboundingbox.go @@ -0,0 +1,173 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/geo" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/search" +) + +func NewGeoBoundingBoxSearcher(indexReader index.IndexReader, minLon, minLat, + maxLon, maxLat float64, field string, boost float64, + options search.SearcherOptions, checkBoundaries bool) ( + search.Searcher, error) { + + // track list of opened searchers, for cleanup on early exit + var openedSearchers []search.Searcher + cleanupOpenedSearchers := func() { + for _, s := range openedSearchers { + _ = s.Close() + } + } + + // do math to produce list of terms needed for this search + onBoundaryTerms, notOnBoundaryTerms := ComputeGeoRange(0, (geo.GeoBits<<1)-1, + minLon, minLat, maxLon, maxLat, checkBoundaries) + + var onBoundarySearcher search.Searcher + if len(onBoundaryTerms) > 0 { + rawOnBoundarySearcher, err := NewMultiTermSearcherBytes(indexReader, + onBoundaryTerms, field, boost, options, false) + if err != nil { + return nil, err + } + // add filter to check points near the boundary + onBoundarySearcher = NewFilteringSearcher(rawOnBoundarySearcher, + buildRectFilter(indexReader, field, minLon, minLat, maxLon, maxLat)) + openedSearchers = append(openedSearchers, onBoundarySearcher) + } + + var notOnBoundarySearcher search.Searcher + if len(notOnBoundaryTerms) > 0 { + var err error + notOnBoundarySearcher, err = NewMultiTermSearcherBytes(indexReader, + notOnBoundaryTerms, field, boost, options, false) + if err != nil { + cleanupOpenedSearchers() + return nil, err + } + openedSearchers = append(openedSearchers, notOnBoundarySearcher) + } + + if onBoundarySearcher != nil && notOnBoundarySearcher != nil { + rv, err := NewDisjunctionSearcher(indexReader, + []search.Searcher{ + onBoundarySearcher, + notOnBoundarySearcher, + }, + 0, options) + if err != nil { + cleanupOpenedSearchers() + return nil, err + } + return rv, nil + } else if onBoundarySearcher != nil { + return onBoundarySearcher, nil + } else if notOnBoundarySearcher != nil { + return notOnBoundarySearcher, nil + } + + return NewMatchNoneSearcher(indexReader) +} + +var geoMaxShift = document.GeoPrecisionStep * 4 +var geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2 + +func ComputeGeoRange(term uint64, shift uint, + sminLon, sminLat, smaxLon, smaxLat float64, + checkBoundaries bool) ( + onBoundary [][]byte, notOnBoundary [][]byte) { + split := term | uint64(0x1)<> 1 + + within := res%document.GeoPrecisionStep == 0 && + geo.RectWithin(minLon, minLat, maxLon, maxLat, + sminLon, sminLat, smaxLon, smaxLat) + if within || (level == geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + sminLon, sminLat, smaxLon, smaxLat)) { + if !within && checkBoundaries { + return [][]byte{ + numeric.MustNewPrefixCodedInt64(int64(start), res), + }, nil + } + return nil, + [][]byte{ + numeric.MustNewPrefixCodedInt64(int64(start), res), + } + } else if level < geoDetailLevel && + geo.RectIntersects(minLon, minLat, maxLon, maxLat, + sminLon, sminLat, smaxLon, smaxLat) { + return ComputeGeoRange(start, res-1, sminLon, sminLat, smaxLon, smaxLat, + checkBoundaries) + } + return nil, nil +} + +func buildRectFilter(indexReader index.IndexReader, field string, + minLon, minLat, maxLon, maxLat float64) FilterFunc { + return func(d *search.DocumentMatch) bool { + var lon, lat float64 + var found bool + err := indexReader.DocumentVisitFieldTerms(d.IndexInternalID, + []string{field}, func(field string, term []byte) { + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + var i64 int64 + i64, err = prefixCoded.Int64() + if err == nil { + lon = geo.MortonUnhashLon(uint64(i64)) + lat = geo.MortonUnhashLat(uint64(i64)) + found = true + } + } + }) + if err == nil && found { + return geo.BoundingBoxContains(lon, lat, + minLon, minLat, maxLon, maxLat) + } + return false + } +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go new file mode 100644 index 00000000..c9d347fa --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go @@ -0,0 +1,115 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/geo" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/search" +) + +func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon, + centerLat, dist float64, field string, boost float64, + options search.SearcherOptions) (search.Searcher, error) { + + // compute bounding box containing the circle + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat := + geo.ComputeBoundingBox(centerLon, centerLat, dist) + + // build a searcher for the box + boxSearcher, err := boxSearcher(indexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, + field, boost, options) + if err != nil { + return nil, err + } + + // wrap it in a filtering searcher which checks the actual distance + return NewFilteringSearcher(boxSearcher, + buildDistFilter(indexReader, field, centerLon, centerLat, dist)), nil +} + +// boxSearcher builds a searcher for the described bounding box +// if the desired box crosses the dateline, it is automatically split into +// two boxes joined through a disjunction searcher +func boxSearcher(indexReader index.IndexReader, + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + if bottomRightLon < topLeftLon { + // cross date line, rewrite as two parts + + leftSearcher, err := NewGeoBoundingBoxSearcher(indexReader, + -180, bottomRightLat, bottomRightLon, topLeftLat, + field, boost, options, false) + if err != nil { + return nil, err + } + rightSearcher, err := NewGeoBoundingBoxSearcher(indexReader, + topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options, false) + if err != nil { + _ = leftSearcher.Close() + return nil, err + } + + boxSearcher, err := NewDisjunctionSearcher(indexReader, + []search.Searcher{leftSearcher, rightSearcher}, 0, options) + if err != nil { + _ = leftSearcher.Close() + _ = rightSearcher.Close() + return nil, err + } + return boxSearcher, nil + } + + // build geoboundinggox searcher for that bounding box + boxSearcher, err := NewGeoBoundingBoxSearcher(indexReader, + topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost, + options, false) + if err != nil { + return nil, err + } + return boxSearcher, nil +} + +func buildDistFilter(indexReader index.IndexReader, field string, + centerLon, centerLat, maxDist float64) FilterFunc { + return func(d *search.DocumentMatch) bool { + var lon, lat float64 + var found bool + err := indexReader.DocumentVisitFieldTerms(d.IndexInternalID, + []string{field}, func(field string, term []byte) { + // only consider the values which are shifted 0 + prefixCoded := numeric.PrefixCoded(term) + shift, err := prefixCoded.Shift() + if err == nil && shift == 0 { + i64, err := prefixCoded.Int64() + if err == nil { + lon = geo.MortonUnhashLon(uint64(i64)) + lat = geo.MortonUnhashLat(uint64(i64)) + found = true + } + } + }) + if err == nil && found { + dist := geo.Haversin(lon, lat, centerLon, centerLat) + if dist <= maxDist/1000 { + return true + } + } + return false + } +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go new file mode 100644 index 00000000..822db2ea --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go @@ -0,0 +1,105 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" +) + +type MatchAllSearcher struct { + indexReader index.IndexReader + reader index.DocIDReader + scorer *scorer.ConstantScorer + count uint64 +} + +func NewMatchAllSearcher(indexReader index.IndexReader, boost float64, options search.SearcherOptions) (*MatchAllSearcher, error) { + reader, err := indexReader.DocIDReaderAll() + if err != nil { + return nil, err + } + count, err := indexReader.DocCount() + if err != nil { + _ = reader.Close() + return nil, err + } + scorer := scorer.NewConstantScorer(1.0, boost, options) + return &MatchAllSearcher{ + indexReader: indexReader, + reader: reader, + scorer: scorer, + count: count, + }, nil +} + +func (s *MatchAllSearcher) Count() uint64 { + return s.count +} + +func (s *MatchAllSearcher) Weight() float64 { + return s.scorer.Weight() +} + +func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *MatchAllSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + id, err := s.reader.Next() + if err != nil { + return nil, err + } + + if id == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, id) + // return doc match + return docMatch, nil + +} + +func (s *MatchAllSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + id, err := s.reader.Advance(ID) + if err != nil { + return nil, err + } + + if id == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, id) + + // return doc match + return docMatch, nil +} + +func (s *MatchAllSearcher) Close() error { + return s.reader.Close() +} + +func (s *MatchAllSearcher) Min() int { + return 0 +} + +func (s *MatchAllSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go new file mode 100644 index 00000000..94759671 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go @@ -0,0 +1,62 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +type MatchNoneSearcher struct { + indexReader index.IndexReader +} + +func NewMatchNoneSearcher(indexReader index.IndexReader) (*MatchNoneSearcher, error) { + return &MatchNoneSearcher{ + indexReader: indexReader, + }, nil +} + +func (s *MatchNoneSearcher) Count() uint64 { + return uint64(0) +} + +func (s *MatchNoneSearcher) Weight() float64 { + return 0.0 +} + +func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) { + +} + +func (s *MatchNoneSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + return nil, nil +} + +func (s *MatchNoneSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + return nil, nil +} + +func (s *MatchNoneSearcher) Close() error { + return nil +} + +func (s *MatchNoneSearcher) Min() int { + return 0 +} + +func (s *MatchNoneSearcher) DocumentMatchPoolSize() int { + return 0 +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go new file mode 100644 index 00000000..b469bead --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_multi_term.go @@ -0,0 +1,85 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +func NewMultiTermSearcher(indexReader index.IndexReader, terms []string, + field string, boost float64, options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + qsearchers := make([]search.Searcher, len(terms)) + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + for i, term := range terms { + var err error + qsearchers[i], err = NewTermSearcher(indexReader, term, field, boost, options) + if err != nil { + qsearchersClose() + return nil, err + } + } + // build disjunction searcher of these ranges + return newMultiTermSearcherBytes(indexReader, qsearchers, field, boost, + options, limit) +} + +func NewMultiTermSearcherBytes(indexReader index.IndexReader, terms [][]byte, + field string, boost float64, options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + qsearchers := make([]search.Searcher, len(terms)) + qsearchersClose := func() { + for _, searcher := range qsearchers { + if searcher != nil { + _ = searcher.Close() + } + } + } + for i, term := range terms { + var err error + qsearchers[i], err = NewTermSearcherBytes(indexReader, term, field, boost, options) + if err != nil { + qsearchersClose() + return nil, err + } + } + return newMultiTermSearcherBytes(indexReader, qsearchers, field, boost, + options, limit) +} + +func newMultiTermSearcherBytes(indexReader index.IndexReader, + searchers []search.Searcher, field string, boost float64, + options search.SearcherOptions, limit bool) ( + search.Searcher, error) { + + // build disjunction searcher of these ranges + searcher, err := newDisjunctionSearcher(indexReader, searchers, 0, options, + limit) + if err != nil { + for _, s := range searchers { + _ = s.Close() + } + return nil, err + } + + return searcher, nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_numeric_range.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go similarity index 53% rename from vendor/github.com/blevesearch/bleve/search/searchers/search_numeric_range.go rename to vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go index 7c8bf9e5..224cc9e2 100644 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_numeric_range.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go @@ -1,33 +1,31 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -package searchers +package searcher import ( "bytes" "math" "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/numeric_util" + "github.com/blevesearch/bleve/numeric" "github.com/blevesearch/bleve/search" ) -type NumericRangeSearcher struct { - indexReader index.IndexReader - min *float64 - max *float64 - field string - explain bool - searcher *DisjunctionSearcher -} - -func NewNumericRangeSearcher(indexReader index.IndexReader, min *float64, max *float64, inclusiveMin, inclusiveMax *bool, field string, boost float64, explain bool) (*NumericRangeSearcher, error) { +func NewNumericRangeSearcher(indexReader index.IndexReader, + min *float64, max *float64, inclusiveMin, inclusiveMax *bool, field string, + boost float64, options search.SearcherOptions) (search.Searcher, error) { // account for unbounded edges if min == nil { negInf := math.Inf(-1) @@ -46,63 +44,23 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, min *float64, max *f inclusiveMax = &defaultInclusiveMax } // find all the ranges - minInt64 := numeric_util.Float64ToInt64(*min) + minInt64 := numeric.Float64ToInt64(*min) if !*inclusiveMin && minInt64 != math.MaxInt64 { minInt64++ } - maxInt64 := numeric_util.Float64ToInt64(*max) + maxInt64 := numeric.Float64ToInt64(*max) if !*inclusiveMax && maxInt64 != math.MinInt64 { maxInt64-- } // FIXME hard-coded precision, should match field declaration termRanges := splitInt64Range(minInt64, maxInt64, 4) terms := termRanges.Enumerate() - // enumerate all the terms in the range - qsearchers := make([]search.Searcher, len(terms)) - for i, term := range terms { - var err error - qsearchers[i], err = NewTermSearcher(indexReader, string(term), field, 1.0, explain) - if err != nil { - return nil, err - } + if tooManyClauses(len(terms)) { + return nil, tooManyClausesErr() } - // build disjunction searcher of these ranges - searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, explain) - if err != nil { - return nil, err - } - return &NumericRangeSearcher{ - indexReader: indexReader, - min: min, - max: max, - field: field, - explain: explain, - searcher: searcher, - }, nil -} -func (s *NumericRangeSearcher) Count() uint64 { - return s.searcher.Count() -} - -func (s *NumericRangeSearcher) Weight() float64 { - return s.searcher.Weight() -} - -func (s *NumericRangeSearcher) SetQueryNorm(qnorm float64) { - s.searcher.SetQueryNorm(qnorm) -} - -func (s *NumericRangeSearcher) Next() (*search.DocumentMatch, error) { - return s.searcher.Next() -} - -func (s *NumericRangeSearcher) Advance(ID string) (*search.DocumentMatch, error) { - return s.searcher.Advance(ID) -} - -func (s *NumericRangeSearcher) Close() error { - return s.searcher.Close() + return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options, + true) } type termRange struct { @@ -111,7 +69,7 @@ type termRange struct { } func (t *termRange) Enumerate() [][]byte { - rv := make([][]byte, 0) + var rv [][]byte next := t.startTerm for bytes.Compare(next, t.endTerm) <= 0 { rv = append(rv, next) @@ -126,7 +84,7 @@ func incrementBytes(in []byte) []byte { for i := len(rv) - 1; i >= 0; i-- { rv[i] = rv[i] + 1 if rv[i] != 0 { - // didnt' overflow, so stop + // didn't overflow, so stop break } } @@ -136,7 +94,7 @@ func incrementBytes(in []byte) []byte { type termRanges []*termRange func (tr termRanges) Enumerate() [][]byte { - rv := make([][]byte, 0) + var rv [][]byte for _, tri := range tr { trie := tri.Enumerate() rv = append(rv, trie...) @@ -173,7 +131,8 @@ func splitInt64Range(minBound, maxBound int64, precisionStep uint) termRanges { lowerWrapped := nextMinBound < minBound upperWrapped := nextMaxBound > maxBound - if shift+precisionStep >= 64 || nextMinBound > nextMaxBound || lowerWrapped || upperWrapped { + if shift+precisionStep >= 64 || nextMinBound > nextMaxBound || + lowerWrapped || upperWrapped { // We are in the lowest precision or the next precision is not available. rv = append(rv, newRange(minBound, maxBound, shift)) // exit the split recursion loop @@ -197,8 +156,8 @@ func splitInt64Range(minBound, maxBound int64, precisionStep uint) termRanges { func newRange(minBound, maxBound int64, shift uint) *termRange { maxBound |= (int64(1) << shift) - int64(1) - minBytes := numeric_util.MustNewPrefixCodedInt64(minBound, shift) - maxBytes := numeric_util.MustNewPrefixCodedInt64(maxBound, shift) + minBytes := numeric.MustNewPrefixCodedInt64(minBound, shift) + maxBytes := numeric.MustNewPrefixCodedInt64(maxBound, shift) return newRangeBytes(minBytes, maxBytes) } @@ -208,7 +167,3 @@ func newRangeBytes(minBytes, maxBytes []byte) *termRange { endTerm: maxBytes, } } - -func (s *NumericRangeSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go new file mode 100644 index 00000000..e3fa0895 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go @@ -0,0 +1,341 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "fmt" + "math" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +type PhraseSearcher struct { + indexReader index.IndexReader + mustSearcher *ConjunctionSearcher + queryNorm float64 + currMust *search.DocumentMatch + slop int + terms [][]string + initialized bool +} + +func NewPhraseSearcher(indexReader index.IndexReader, terms []string, field string, options search.SearcherOptions) (*PhraseSearcher, error) { + // turn flat terms []string into [][]string + mterms := make([][]string, len(terms)) + for i, term := range terms { + mterms[i] = []string{term} + } + return NewMultiPhraseSearcher(indexReader, mterms, field, options) +} + +func NewMultiPhraseSearcher(indexReader index.IndexReader, terms [][]string, field string, options search.SearcherOptions) (*PhraseSearcher, error) { + options.IncludeTermVectors = true + var termPositionSearchers []search.Searcher + for _, termPos := range terms { + if len(termPos) == 1 && termPos[0] != "" { + // single term + ts, err := NewTermSearcher(indexReader, termPos[0], field, 1.0, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) + } + termPositionSearchers = append(termPositionSearchers, ts) + } else if len(termPos) > 1 { + // multiple terms + var termSearchers []search.Searcher + for _, term := range termPos { + if term == "" { + continue + } + ts, err := NewTermSearcher(indexReader, term, field, 1.0, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err) + } + termSearchers = append(termSearchers, ts) + } + disjunction, err := NewDisjunctionSearcher(indexReader, termSearchers, 1, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building term position disjunction searcher: %v", err) + } + termPositionSearchers = append(termPositionSearchers, disjunction) + } + } + + mustSearcher, err := NewConjunctionSearcher(indexReader, termPositionSearchers, options) + if err != nil { + // close any searchers already opened + for _, ts := range termPositionSearchers { + _ = ts.Close() + } + return nil, fmt.Errorf("phrase searcher error building conjunction searcher: %v", err) + } + + // build our searcher + rv := PhraseSearcher{ + indexReader: indexReader, + mustSearcher: mustSearcher, + terms: terms, + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *PhraseSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + if s.mustSearcher != nil { + sumOfSquaredWeights += s.mustSearcher.Weight() + } + + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + if s.mustSearcher != nil { + s.mustSearcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error { + err := s.advanceNextMust(ctx) + if err != nil { + return err + } + + s.initialized = true + return nil +} + +func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error { + var err error + + if s.mustSearcher != nil { + s.currMust, err = s.mustSearcher.Next(ctx) + if err != nil { + return err + } + } + + return nil +} + +func (s *PhraseSearcher) Weight() float64 { + return s.mustSearcher.Weight() +} + +func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { + s.mustSearcher.SetQueryNorm(qnorm) +} + +func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + for s.currMust != nil { + // check this match against phrase constraints + rv := s.checkCurrMustMatch(ctx) + + // prepare for next iteration (either loop or subsequent call to Next()) + err := s.advanceNextMust(ctx) + if err != nil { + return nil, err + } + + // if match satisfied phrase constraints return it as a hit + if rv != nil { + return rv, nil + } + } + + return nil, nil +} + +// checkCurrMustMatch is soley concerned with determining if the DocumentMatch +// pointed to by s.currMust (which satisifies the pre-condition searcher) +// also satisfies the phase constraints. if so, it returns a DocumentMatch +// for this document, otherwise nil +func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch { + rvftlm := make(search.FieldTermLocationMap, 0) + freq := 0 + // typically we would expect there to only actually be results in + // one field, but we allow for this to not be the case + // but, we note that phrase constraints can only be satisfied within + // a single field, so we can check them each independently + for field, tlm := range s.currMust.Locations { + + f, rvtlm := s.checkCurrMustMatchField(ctx, tlm) + if f > 0 { + freq += f + rvftlm[field] = rvtlm + } + } + + if freq > 0 { + // return match + rv := s.currMust + rv.Locations = rvftlm + return rv + } + + return nil +} + +// checkCurrMustMatchField is soley concerned with determining if one particular +// field within the currMust DocumentMatch Locations satisfies the phase +// constraints (possibly more than once). if so, the number of times it was +// satisfied, and these locations are returned. otherwise 0 and either +// a nil or empty TermLocationMap +func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, tlm search.TermLocationMap) (int, search.TermLocationMap) { + paths := findPhrasePaths(0, nil, s.terms, tlm, nil, 0) + rv := make(search.TermLocationMap, len(s.terms)) + for _, p := range paths { + p.MergeInto(rv) + } + return len(paths), rv +} + +type phrasePart struct { + term string + loc *search.Location +} + +type phrasePath []*phrasePart + +func (p phrasePath) MergeInto(in search.TermLocationMap) { + for _, pp := range p { + in[pp.term] = append(in[pp.term], pp.loc) + } +} + +// findPhrasePaths is a function to identify phase matches from a set of known +// term locations. the implementation is recursive, so care must be taken +// with arguments and return values. +// +// prev - the previous location, nil on first invocation +// phraseTerms - slice containing the phrase terms themselves +// may contain empty string as placeholder (don't care) +// tlm - the Term Location Map containing all relevant term locations +// offset - the offset from the previous that this next term must match +// p - the current path being explored (appended to in recursive calls) +// this is the primary state being built during the traversal +// +// returns slice of paths, or nil if invocation did not find any successul paths +func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, tlm search.TermLocationMap, p phrasePath, remainingSlop int) []phrasePath { + + // no more terms + if len(phraseTerms) < 1 { + return []phrasePath{p} + } + + car := phraseTerms[0] + cdr := phraseTerms[1:] + + // empty term is treated as match (continue) + if len(car) == 0 || (len(car) == 1 && car[0] == "") { + nextPos := prevPos + 1 + if prevPos == 0 { + // if prevPos was 0, don't set it to 1 (as thats not a real abs pos) + nextPos = 0 // don't advance nextPos if prevPos was 0 + } + return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop) + } + + var rv []phrasePath + // locations for this term + for _, carTerm := range car { + locations := tlm[carTerm] + for _, loc := range locations { + if prevPos != 0 && !loc.ArrayPositions.Equals(ap) { + // if the array positions are wrong, can't match, try next location + continue + } + + // compute distance from previous phrase term + dist := 0 + if prevPos != 0 { + dist = editDistance(prevPos+1, loc.Pos) + } + + // if enough slop reamining, continue recursively + if prevPos == 0 || (remainingSlop-dist) >= 0 { + // this location works, add it to the path (but not for empty term) + px := append(p, &phrasePart{term: carTerm, loc: loc}) + rv = append(rv, findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist)...) + } + } + } + return rv +} + +func editDistance(p1, p2 uint64) int { + dist := int(p1 - p2) + if dist < 0 { + return -dist + } + return dist +} + +func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + var err error + s.currMust, err = s.mustSearcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + return s.Next(ctx) +} + +func (s *PhraseSearcher) Count() uint64 { + // for now return a worst case + return s.mustSearcher.Count() +} + +func (s *PhraseSearcher) Close() error { + if s.mustSearcher != nil { + err := s.mustSearcher.Close() + if err != nil { + return err + } + } + return nil +} + +func (s *PhraseSearcher) Min() int { + return 0 +} + +func (s *PhraseSearcher) DocumentMatchPoolSize() int { + return s.mustSearcher.DocumentMatchPoolSize() + 1 +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go new file mode 100644 index 00000000..b7cf520a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go @@ -0,0 +1,80 @@ +// Copyright (c) 2015 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "regexp" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +// NewRegexpSearcher creates a searcher which will match documents that +// contain terms which match the pattern regexp. The match must be EXACT +// matching the entire term. The provided regexp SHOULD NOT start with ^ +// or end with $ as this can intefere with the implementation. Separately, +// matches will be checked to ensure they match the entire term. +func NewRegexpSearcher(indexReader index.IndexReader, pattern *regexp.Regexp, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + + prefixTerm, complete := pattern.LiteralPrefix() + var candidateTerms []string + if complete { + // there is no pattern + candidateTerms = []string{prefixTerm} + } else { + var err error + candidateTerms, err = findRegexpCandidateTerms(indexReader, pattern, field, + prefixTerm) + if err != nil { + return nil, err + } + } + + return NewMultiTermSearcher(indexReader, candidateTerms, field, boost, + options, true) +} + +func findRegexpCandidateTerms(indexReader index.IndexReader, + pattern *regexp.Regexp, field, prefixTerm string) (rv []string, err error) { + rv = make([]string, 0) + var fieldDict index.FieldDict + if len(prefixTerm) > 0 { + fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) + } else { + fieldDict, err = indexReader.FieldDict(field) + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // enumerate the terms and check against regexp + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + matchPos := pattern.FindStringIndex(tfd.Term) + if matchPos != nil && matchPos[0] == 0 && matchPos[1] == len(tfd.Term) { + rv = append(rv, tfd.Term) + if tooManyClauses(len(rv)) { + return rv, tooManyClausesErr() + } + } + tfd, err = fieldDict.Next() + } + + return rv, err +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go new file mode 100644 index 00000000..6fae6ae5 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go @@ -0,0 +1,122 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" +) + +type TermSearcher struct { + indexReader index.IndexReader + reader index.TermFieldReader + scorer *scorer.TermQueryScorer + tfd index.TermFieldDoc +} + +func NewTermSearcher(indexReader index.IndexReader, term string, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { + reader, err := indexReader.TermFieldReader([]byte(term), field, true, true, options.IncludeTermVectors) + if err != nil { + return nil, err + } + count, err := indexReader.DocCount() + if err != nil { + _ = reader.Close() + return nil, err + } + scorer := scorer.NewTermQueryScorer([]byte(term), field, boost, count, reader.Count(), options) + return &TermSearcher{ + indexReader: indexReader, + reader: reader, + scorer: scorer, + }, nil +} + +func NewTermSearcherBytes(indexReader index.IndexReader, term []byte, field string, boost float64, options search.SearcherOptions) (*TermSearcher, error) { + reader, err := indexReader.TermFieldReader(term, field, true, true, options.IncludeTermVectors) + if err != nil { + return nil, err + } + count, err := indexReader.DocCount() + if err != nil { + _ = reader.Close() + return nil, err + } + scorer := scorer.NewTermQueryScorer(term, field, boost, count, reader.Count(), options) + return &TermSearcher{ + indexReader: indexReader, + reader: reader, + scorer: scorer, + }, nil +} + +func (s *TermSearcher) Count() uint64 { + return s.reader.Count() +} + +func (s *TermSearcher) Weight() float64 { + return s.scorer.Weight() +} + +func (s *TermSearcher) SetQueryNorm(qnorm float64) { + s.scorer.SetQueryNorm(qnorm) +} + +func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { + termMatch, err := s.reader.Next(s.tfd.Reset()) + if err != nil { + return nil, err + } + + if termMatch == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, termMatch) + // return doc match + return docMatch, nil + +} + +func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) { + termMatch, err := s.reader.Advance(ID, s.tfd.Reset()) + if err != nil { + return nil, err + } + + if termMatch == nil { + return nil, nil + } + + // score match + docMatch := s.scorer.Score(ctx, termMatch) + + // return doc match + return docMatch, nil +} + +func (s *TermSearcher) Close() error { + return s.reader.Close() +} + +func (s *TermSearcher) Min() int { + return 0 +} + +func (s *TermSearcher) DocumentMatchPoolSize() int { + return 1 +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go new file mode 100644 index 00000000..05d09224 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_term_prefix.go @@ -0,0 +1,39 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +func NewTermPrefixSearcher(indexReader index.IndexReader, prefix string, + field string, boost float64, options search.SearcherOptions) ( + search.Searcher, error) { + // find the terms with this prefix + fieldDict, err := indexReader.FieldDictPrefix(field, []byte(prefix)) + if err != nil { + return nil, err + } + + var terms []string + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + terms = append(terms, tfd.Term) + tfd, err = fieldDict.Next() + } + + return NewMultiTermSearcher(indexReader, terms, field, boost, options, true) +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go new file mode 100644 index 00000000..24170c71 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" +) + +func NewTermRangeSearcher(indexReader index.IndexReader, + min, max []byte, inclusiveMin, inclusiveMax *bool, field string, + boost float64, options search.SearcherOptions) (search.Searcher, error) { + + if inclusiveMin == nil { + defaultInclusiveMin := true + inclusiveMin = &defaultInclusiveMin + } + if inclusiveMax == nil { + defaultInclusiveMax := false + inclusiveMax = &defaultInclusiveMax + } + + if min == nil { + min = []byte{} + } + + rangeMax := max + if rangeMax != nil { + // the term dictionary range end has an unfortunate implementation + rangeMax = append(rangeMax, 0) + } + + // find the terms with this prefix + fieldDict, err := indexReader.FieldDictRange(field, min, rangeMax) + if err != nil { + return nil, err + } + + var terms []string + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + terms = append(terms, tfd.Term) + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + + if len(terms) < 1 { + return NewMatchNoneSearcher(indexReader) + } + + if !*inclusiveMin && min != nil && string(min) == terms[0] { + terms = terms[1:] + } + + // if our term list included the max, it would be the last item + if !*inclusiveMax && max != nil && string(max) == terms[len(terms)-1] { + terms = terms[:len(terms)-1] + } + + return NewMultiTermSearcher(indexReader, terms, field, boost, options, true) +} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/base_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/base_test.go deleted file mode 100644 index 98945c65..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/base_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "math" - "regexp" - - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/analysis/tokenizers/regexp_tokenizer" - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/index/store/inmem" - "github.com/blevesearch/bleve/index/upside_down" -) - -var twoDocIndex index.Index //= upside_down.NewUpsideDownCouch(inmem.MustOpen()) - -func init() { - inMemStore, _ := inmem.New() - analysisQueue := upside_down.NewAnalysisQueue(1) - twoDocIndex = upside_down.NewUpsideDownCouch(inMemStore, analysisQueue) - err := twoDocIndex.Open() - if err != nil { - panic(err) - } - for _, doc := range twoDocIndexDocs { - err := twoDocIndex.Update(doc) - if err != nil { - panic(err) - } - } -} - -// create a simpler analyzer which will support these tests -var testAnalyzer = &analysis.Analyzer{ - Tokenizer: regexp_tokenizer.NewRegexpTokenizer(regexp.MustCompile(`\w+`)), -} - -// sets up some mock data used in many tests in this package -var twoDocIndexDescIndexingOptions = document.DefaultTextIndexingOptions | document.IncludeTermVectors - -var twoDocIndexDocs = []*document.Document{ - // must have 4/4 beer - document.NewDocument("1"). - AddField(document.NewTextField("name", []uint64{}, []byte("marty"))). - AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("beer beer beer beer"), twoDocIndexDescIndexingOptions, testAnalyzer)). - AddField(document.NewTextFieldWithAnalyzer("street", []uint64{}, []byte("couchbase way"), testAnalyzer)), - // must have 1/4 beer - document.NewDocument("2"). - AddField(document.NewTextField("name", []uint64{}, []byte("steve"))). - AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("angst beer couch database"), twoDocIndexDescIndexingOptions, testAnalyzer)). - AddField(document.NewTextFieldWithAnalyzer("street", []uint64{}, []byte("couchbase way"), testAnalyzer)). - AddField(document.NewTextFieldWithAnalyzer("title", []uint64{}, []byte("mister"), testAnalyzer)), - // must have 1/4 beer - document.NewDocument("3"). - AddField(document.NewTextField("name", []uint64{}, []byte("dustin"))). - AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("apple beer column dank"), twoDocIndexDescIndexingOptions, testAnalyzer)). - AddField(document.NewTextFieldWithAnalyzer("title", []uint64{}, []byte("mister"), testAnalyzer)), - // must have 65/65 beer - document.NewDocument("4"). - AddField(document.NewTextField("name", []uint64{}, []byte("ravi"))). - AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer beer"), twoDocIndexDescIndexingOptions, testAnalyzer)), - // must have 0/x beer - document.NewDocument("5"). - AddField(document.NewTextField("name", []uint64{}, []byte("bobert"))). - AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("water"), twoDocIndexDescIndexingOptions, testAnalyzer)). - AddField(document.NewTextFieldWithAnalyzer("title", []uint64{}, []byte("mister"), testAnalyzer)), -} - -func scoresCloseEnough(a, b float64) bool { - return math.Abs(a-b) < 0.001 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/ordered_searchers_list.go b/vendor/github.com/blevesearch/bleve/search/searchers/ordered_searchers_list.go deleted file mode 100644 index 34e06a43..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/ordered_searchers_list.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "github.com/blevesearch/bleve/search" -) - -type OrderedSearcherList []search.Searcher - -// sort.Interface - -func (otrl OrderedSearcherList) Len() int { - return len(otrl) -} - -func (otrl OrderedSearcherList) Less(i, j int) bool { - return otrl[i].Count() < otrl[j].Count() -} - -func (otrl OrderedSearcherList) Swap(i, j int) { - otrl[i], otrl[j] = otrl[j], otrl[i] -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_boolean.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_boolean.go deleted file mode 100644 index bf44e027..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_boolean.go +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "math" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorers" -) - -type BooleanSearcher struct { - initialized bool - indexReader index.IndexReader - mustSearcher search.Searcher - shouldSearcher search.Searcher - mustNotSearcher search.Searcher - queryNorm float64 - currMust *search.DocumentMatch - currShould *search.DocumentMatch - currMustNot *search.DocumentMatch - currentID string - min uint64 - scorer *scorers.ConjunctionQueryScorer -} - -func NewBooleanSearcher(indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, explain bool) (*BooleanSearcher, error) { - // build our searcher - rv := BooleanSearcher{ - indexReader: indexReader, - mustSearcher: mustSearcher, - shouldSearcher: shouldSearcher, - mustNotSearcher: mustNotSearcher, - scorer: scorers.NewConjunctionQueryScorer(explain), - } - rv.computeQueryNorm() - return &rv, nil -} - -func (s *BooleanSearcher) computeQueryNorm() { - // first calculate sum of squared weights - sumOfSquaredWeights := 0.0 - if s.mustSearcher != nil { - sumOfSquaredWeights += s.mustSearcher.Weight() - } - if s.shouldSearcher != nil { - sumOfSquaredWeights += s.shouldSearcher.Weight() - } - - // now compute query norm from this - s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) - // finally tell all the downstream searchers the norm - if s.mustSearcher != nil { - s.mustSearcher.SetQueryNorm(s.queryNorm) - } - if s.shouldSearcher != nil { - s.shouldSearcher.SetQueryNorm(s.queryNorm) - } -} - -func (s *BooleanSearcher) initSearchers() error { - var err error - // get all searchers pointing at their first match - if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next() - if err != nil { - return err - } - } - - if s.shouldSearcher != nil { - s.currShould, err = s.shouldSearcher.Next() - if err != nil { - return err - } - } - - if s.mustNotSearcher != nil { - s.currMustNot, err = s.mustNotSearcher.Next() - if err != nil { - return err - } - } - - if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID - } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID - } else { - s.currentID = "" - } - - s.initialized = true - return nil -} - -func (s *BooleanSearcher) advanceNextMust() error { - var err error - - if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next() - if err != nil { - return err - } - } else if s.mustSearcher == nil { - s.currShould, err = s.shouldSearcher.Next() - if err != nil { - return err - } - } - - if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID - } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID - } else { - s.currentID = "" - } - return nil -} - -func (s *BooleanSearcher) Weight() float64 { - var rv float64 - if s.mustSearcher != nil { - rv += s.mustSearcher.Weight() - } - if s.shouldSearcher != nil { - rv += s.shouldSearcher.Weight() - } - - return rv -} - -func (s *BooleanSearcher) SetQueryNorm(qnorm float64) { - if s.mustSearcher != nil { - s.mustSearcher.SetQueryNorm(qnorm) - } - if s.shouldSearcher != nil { - s.shouldSearcher.SetQueryNorm(qnorm) - } -} - -func (s *BooleanSearcher) Next() (*search.DocumentMatch, error) { - - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - - var err error - var rv *search.DocumentMatch - - for s.currentID != "" { - if s.currMustNot != nil && s.currMustNot.ID < s.currentID { - // advance must not searcher to our candidate entry - s.currMustNot, err = s.mustNotSearcher.Advance(s.currentID) - if err != nil { - return nil, err - } - if s.currMustNot != nil && s.currMustNot.ID == s.currentID { - // the candidate is excluded - err = s.advanceNextMust() - if err != nil { - return nil, err - } - continue - } - } else if s.currMustNot != nil && s.currMustNot.ID == s.currentID { - // the candidate is excluded - err = s.advanceNextMust() - if err != nil { - return nil, err - } - continue - } - - if s.currShould != nil && s.currShould.ID < s.currentID { - // advance should searcher to our candidate entry - s.currShould, err = s.shouldSearcher.Advance(s.currentID) - if err != nil { - return nil, err - } - if s.currShould != nil && s.currShould.ID == s.currentID { - // score bonus matches should - var cons []*search.DocumentMatch - if s.currMust != nil { - cons = []*search.DocumentMatch{ - s.currMust, - s.currShould, - } - } else { - cons = []*search.DocumentMatch{ - s.currShould, - } - } - rv = s.scorer.Score(cons) - err = s.advanceNextMust() - if err != nil { - return nil, err - } - break - } else if s.shouldSearcher.Min() == 0 { - // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) - err = s.advanceNextMust() - if err != nil { - return nil, err - } - break - } - } else if s.currShould != nil && s.currShould.ID == s.currentID { - // score bonus matches should - var cons []*search.DocumentMatch - if s.currMust != nil { - cons = []*search.DocumentMatch{ - s.currMust, - s.currShould, - } - } else { - cons = []*search.DocumentMatch{ - s.currShould, - } - } - rv = s.scorer.Score(cons) - err = s.advanceNextMust() - if err != nil { - return nil, err - } - break - } else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 { - // match is OK anyway - rv = s.scorer.Score([]*search.DocumentMatch{s.currMust}) - err = s.advanceNextMust() - if err != nil { - return nil, err - } - break - } - - err = s.advanceNextMust() - if err != nil { - return nil, err - } - } - return rv, nil -} - -func (s *BooleanSearcher) Advance(ID string) (*search.DocumentMatch, error) { - - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - - var err error - if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Advance(ID) - if err != nil { - return nil, err - } - } - if s.shouldSearcher != nil { - s.currShould, err = s.shouldSearcher.Advance(ID) - if err != nil { - return nil, err - } - } - if s.mustNotSearcher != nil { - s.currMustNot, err = s.mustNotSearcher.Advance(ID) - if err != nil { - return nil, err - } - } - - if s.mustSearcher != nil && s.currMust != nil { - s.currentID = s.currMust.ID - } else if s.mustSearcher == nil && s.currShould != nil { - s.currentID = s.currShould.ID - } else { - s.currentID = "" - } - - return s.Next() -} - -func (s *BooleanSearcher) Count() uint64 { - - // for now return a worst case - var sum uint64 - if s.mustSearcher != nil { - sum += s.mustSearcher.Count() - } - if s.shouldSearcher != nil { - sum += s.shouldSearcher.Count() - } - return sum -} - -func (s *BooleanSearcher) Close() error { - if s.mustSearcher != nil { - err := s.mustSearcher.Close() - if err != nil { - return err - } - } - if s.shouldSearcher != nil { - err := s.shouldSearcher.Close() - if err != nil { - return err - } - } - if s.mustNotSearcher != nil { - err := s.mustNotSearcher.Close() - if err != nil { - return err - } - } - return nil -} - -func (s *BooleanSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_boolean_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_boolean_test.go deleted file mode 100644 index 875424b9..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_boolean_test.go +++ /dev/null @@ -1,364 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestBooleanSearch(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // test 0 - beerTermSearcher, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher}, true) - if err != nil { - t.Fatal(err) - } - martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - shouldSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, true) - if err != nil { - t.Fatal(err) - } - steveTermSearcher, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustNotSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher}, 0, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher, shouldSearcher, mustNotSearcher, true) - if err != nil { - t.Fatal(err) - } - - // test 1 - martyTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - shouldSearcher2, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, true) - if err != nil { - t.Fatal(err) - } - steveTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustNotSearcher2, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher2}, 0, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher2, err := NewBooleanSearcher(twoDocIndexReader, nil, shouldSearcher2, mustNotSearcher2, true) - if err != nil { - t.Fatal(err) - } - - // test 2 - steveTermSearcher3, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustNotSearcher3, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher3}, 0, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher3, err := NewBooleanSearcher(twoDocIndexReader, nil, nil, mustNotSearcher3, true) - if err != nil { - t.Fatal(err) - } - - // test 3 - beerTermSearcher4, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustSearcher4, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher4}, true) - if err != nil { - t.Fatal(err) - } - steveTermSearcher4, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustNotSearcher4, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher4}, 0, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher4, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher4, nil, mustNotSearcher4, true) - if err != nil { - t.Fatal(err) - } - - // test 4 - beerTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustSearcher5, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher5}, true) - if err != nil { - t.Fatal(err) - } - steveTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - martyTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustNotSearcher5, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher5, martyTermSearcher5}, 0, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher5, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher5, nil, mustNotSearcher5, true) - if err != nil { - t.Fatal(err) - } - - // test 5 - beerTermSearcher6, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustSearcher6, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher6}, true) - if err != nil { - t.Fatal(err) - } - martyTermSearcher6, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher6, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - shouldSearcher6, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher6, dustinTermSearcher6}, 2, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher6, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher6, shouldSearcher6, nil, true) - if err != nil { - t.Fatal(err) - } - - // test 6 - beerTermSearcher7, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustSearcher7, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher7}, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher7, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher7, nil, nil, true) - if err != nil { - t.Fatal(err) - } - martyTermSearcher7, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 5.0, true) - if err != nil { - t.Fatal(err) - } - conjunctionSearcher7, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher7, booleanSearcher7}, true) - - // test 7 - beerTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustSearcher8, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher8}, true) - if err != nil { - t.Fatal(err) - } - martyTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - shouldSearcher8, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher8, dustinTermSearcher8}, 0, true) - if err != nil { - t.Fatal(err) - } - steveTermSearcher8, err := NewTermSearcher(twoDocIndexReader, "steve", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustNotSearcher8, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{steveTermSearcher8}, 0, true) - if err != nil { - t.Fatal(err) - } - booleanSearcher8, err := NewBooleanSearcher(twoDocIndexReader, mustSearcher8, shouldSearcher8, mustNotSearcher8, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher8a, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 5.0, true) - if err != nil { - t.Fatal(err) - } - conjunctionSearcher8, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{booleanSearcher8, dustinTermSearcher8a}, true) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - searcher search.Searcher - results []*search.DocumentMatch - }{ - { - searcher: booleanSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 0.9818005051949021, - }, - &search.DocumentMatch{ - ID: "3", - Score: 0.808709699395535, - }, - &search.DocumentMatch{ - ID: "4", - Score: 0.34618161159873423, - }, - }, - }, - { - searcher: booleanSearcher2, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 0.6775110856165737, - }, - &search.DocumentMatch{ - ID: "3", - Score: 0.6775110856165737, - }, - }, - }, - // no MUST or SHOULD clauses yields no results - { - searcher: booleanSearcher3, - results: []*search.DocumentMatch{}, - }, - { - searcher: booleanSearcher4, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "3", - Score: 0.5, - }, - &search.DocumentMatch{ - ID: "4", - Score: 1.0, - }, - }, - }, - { - searcher: booleanSearcher5, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "3", - Score: 0.5, - }, - &search.DocumentMatch{ - ID: "4", - Score: 1.0, - }, - }, - }, - { - searcher: booleanSearcher6, - results: []*search.DocumentMatch{}, - }, - // test a conjunction query with a nested boolean - { - searcher: conjunctionSearcher7, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 2.0097428702814377, - }, - }, - }, - { - searcher: conjunctionSearcher8, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "3", - Score: 2.0681575785068107, - }, - }, - }, - } - - for testIndex, test := range tests { - defer func() { - err := test.searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - next, err := test.searcher.Next() - i := 0 - for err == nil && next != nil { - if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) - } - if !scoresCloseEnough(next.Score, test.results[i].Score) { - t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) - t.Logf("scoring explanation: %s", next.Expl) - } - } - next, err = test.searcher.Next() - i++ - } - if err != nil { - t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) - } - if len(test.results) != i { - t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction.go deleted file mode 100644 index 77c4b1ff..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "math" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorers" -) - -type ConjunctionSearcher struct { - initialized bool - indexReader index.IndexReader - searchers OrderedSearcherList - explain bool - queryNorm float64 - currs []*search.DocumentMatch - currentID string - scorer *scorers.ConjunctionQueryScorer -} - -func NewConjunctionSearcher(indexReader index.IndexReader, qsearchers []search.Searcher, explain bool) (*ConjunctionSearcher, error) { - // build the downstream searchers - searchers := make(OrderedSearcherList, len(qsearchers)) - for i, searcher := range qsearchers { - searchers[i] = searcher - } - // sort the searchers - sort.Sort(searchers) - // build our searcher - rv := ConjunctionSearcher{ - indexReader: indexReader, - explain: explain, - searchers: searchers, - currs: make([]*search.DocumentMatch, len(searchers)), - scorer: scorers.NewConjunctionQueryScorer(explain), - } - rv.computeQueryNorm() - return &rv, nil -} - -func (s *ConjunctionSearcher) computeQueryNorm() { - // first calculate sum of squared weights - sumOfSquaredWeights := 0.0 - for _, termSearcher := range s.searchers { - sumOfSquaredWeights += termSearcher.Weight() - } - // now compute query norm from this - s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) - // finally tell all the downstream searchers the norm - for _, termSearcher := range s.searchers { - termSearcher.SetQueryNorm(s.queryNorm) - } -} - -func (s *ConjunctionSearcher) initSearchers() error { - var err error - // get all searchers pointing at their first match - for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Next() - if err != nil { - return err - } - } - - if len(s.currs) > 0 { - if s.currs[0] != nil { - s.currentID = s.currs[0].ID - } else { - s.currentID = "" - } - } - - s.initialized = true - return nil -} - -func (s *ConjunctionSearcher) Weight() float64 { - var rv float64 - for _, searcher := range s.searchers { - rv += searcher.Weight() - } - return rv -} - -func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) { - for _, searcher := range s.searchers { - searcher.SetQueryNorm(qnorm) - } -} - -func (s *ConjunctionSearcher) Next() (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - var rv *search.DocumentMatch - var err error -OUTER: - for s.currentID != "" { - for i, termSearcher := range s.searchers { - if s.currs[i] != nil && s.currs[i].ID != s.currentID { - if s.currentID < s.currs[i].ID { - s.currentID = s.currs[i].ID - continue OUTER - } - // this reader doesn't have the currentID, try to advance - s.currs[i], err = termSearcher.Advance(s.currentID) - if err != nil { - return nil, err - } - if s.currs[i] == nil { - s.currentID = "" - continue OUTER - } - if s.currs[i].ID != s.currentID { - // we just advanced, so it doesn't match, it must be greater - // no need to call next - s.currentID = s.currs[i].ID - continue OUTER - } - } else if s.currs[i] == nil { - s.currentID = "" - continue OUTER - } - } - // if we get here, a doc matched all readers, sum the score and add it - rv = s.scorer.Score(s.currs) - - // prepare for next entry - s.currs[0], err = s.searchers[0].Next() - if err != nil { - return nil, err - } - if s.currs[0] == nil { - s.currentID = "" - } else { - s.currentID = s.currs[0].ID - } - // don't continue now, wait for the next call to Next() - break - } - return rv, nil -} - -func (s *ConjunctionSearcher) Advance(ID string) (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - var err error - for i, searcher := range s.searchers { - s.currs[i], err = searcher.Advance(ID) - if err != nil { - return nil, err - } - } - s.currentID = ID - return s.Next() -} - -func (s *ConjunctionSearcher) Count() uint64 { - // for now return a worst case - var sum uint64 - for _, searcher := range s.searchers { - sum += searcher.Count() - } - return sum -} - -func (s *ConjunctionSearcher) Close() error { - for _, searcher := range s.searchers { - err := searcher.Close() - if err != nil { - return err - } - } - return nil -} - -func (s *ConjunctionSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction_test.go deleted file mode 100644 index 747df272..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_conjunction_test.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestConjunctionSearch(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - // test 0 - beerTermSearcher, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 5.0, true) - if err != nil { - t.Fatal(err) - } - beerAndMartySearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher, martyTermSearcher}, true) - if err != nil { - t.Fatal(err) - } - - // test 1 - angstTermSearcher, err := NewTermSearcher(twoDocIndexReader, "angst", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - beerTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - angstAndBeerSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{angstTermSearcher, beerTermSearcher2}, true) - if err != nil { - t.Fatal(err) - } - - // test 2 - beerTermSearcher3, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - jackTermSearcher, err := NewTermSearcher(twoDocIndexReader, "jack", "name", 5.0, true) - if err != nil { - t.Fatal(err) - } - beerAndJackSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher3, jackTermSearcher}, true) - if err != nil { - t.Fatal(err) - } - - // test 3 - beerTermSearcher4, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - misterTermSearcher, err := NewTermSearcher(twoDocIndexReader, "mister", "title", 5.0, true) - if err != nil { - t.Fatal(err) - } - beerAndMisterSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher4, misterTermSearcher}, true) - if err != nil { - t.Fatal(err) - } - - // test 4 - couchbaseTermSearcher, err := NewTermSearcher(twoDocIndexReader, "couchbase", "street", 1.0, true) - if err != nil { - t.Fatal(err) - } - misterTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "mister", "title", 5.0, true) - if err != nil { - t.Fatal(err) - } - couchbaseAndMisterSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{couchbaseTermSearcher, misterTermSearcher2}, true) - if err != nil { - t.Fatal(err) - } - - // test 5 - beerTermSearcher5, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 5.0, true) - if err != nil { - t.Fatal(err) - } - couchbaseTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "couchbase", "street", 1.0, true) - if err != nil { - t.Fatal(err) - } - misterTermSearcher3, err := NewTermSearcher(twoDocIndexReader, "mister", "title", 5.0, true) - if err != nil { - t.Fatal(err) - } - couchbaseAndMisterSearcher2, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{couchbaseTermSearcher2, misterTermSearcher3}, true) - if err != nil { - t.Fatal(err) - } - beerAndCouchbaseAndMisterSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{beerTermSearcher5, couchbaseAndMisterSearcher2}, true) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - searcher search.Searcher - results []*search.DocumentMatch - }{ - { - searcher: beerAndMartySearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 2.0097428702814377, - }, - }, - }, - { - searcher: angstAndBeerSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "2", - Score: 1.0807601687084403, - }, - }, - }, - { - searcher: beerAndJackSearcher, - results: []*search.DocumentMatch{}, - }, - { - searcher: beerAndMisterSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "2", - Score: 1.2877980334016337, - }, - &search.DocumentMatch{ - ID: "3", - Score: 1.2877980334016337, - }, - }, - }, - { - searcher: couchbaseAndMisterSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "2", - Score: 1.4436599157093672, - }, - }, - }, - { - searcher: beerAndCouchbaseAndMisterSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "2", - Score: 1.441614953806971, - }, - }, - }, - } - - for testIndex, test := range tests { - defer func() { - err := test.searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - next, err := test.searcher.Next() - i := 0 - for err == nil && next != nil { - if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) - } - if !scoresCloseEnough(next.Score, test.results[i].Score) { - t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) - t.Logf("scoring explanation: %s", next.Expl) - } - } - next, err = test.searcher.Next() - i++ - } - if err != nil { - t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) - } - if len(test.results) != i { - t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction.go deleted file mode 100644 index 652ca395..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "math" - "sort" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorers" -) - -type DisjunctionSearcher struct { - initialized bool - indexReader index.IndexReader - searchers OrderedSearcherList - queryNorm float64 - currs []*search.DocumentMatch - currentID string - scorer *scorers.DisjunctionQueryScorer - min float64 -} - -func NewDisjunctionSearcher(indexReader index.IndexReader, qsearchers []search.Searcher, min float64, explain bool) (*DisjunctionSearcher, error) { - // build the downstream searchers - searchers := make(OrderedSearcherList, len(qsearchers)) - for i, searcher := range qsearchers { - searchers[i] = searcher - } - // sort the searchers - sort.Sort(sort.Reverse(searchers)) - // build our searcher - rv := DisjunctionSearcher{ - indexReader: indexReader, - searchers: searchers, - currs: make([]*search.DocumentMatch, len(searchers)), - scorer: scorers.NewDisjunctionQueryScorer(explain), - min: min, - } - rv.computeQueryNorm() - return &rv, nil -} - -func (s *DisjunctionSearcher) computeQueryNorm() { - // first calculate sum of squared weights - sumOfSquaredWeights := 0.0 - for _, termSearcher := range s.searchers { - sumOfSquaredWeights += termSearcher.Weight() - } - // now compute query norm from this - s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) - // finally tell all the downstream searchers the norm - for _, termSearcher := range s.searchers { - termSearcher.SetQueryNorm(s.queryNorm) - } -} - -func (s *DisjunctionSearcher) initSearchers() error { - var err error - // get all searchers pointing at their first match - for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Next() - if err != nil { - return err - } - } - - s.currentID = s.nextSmallestID() - s.initialized = true - return nil -} - -func (s *DisjunctionSearcher) nextSmallestID() string { - rv := "" - for _, curr := range s.currs { - if curr != nil && (curr.ID < rv || rv == "") { - rv = curr.ID - } - } - return rv -} - -func (s *DisjunctionSearcher) Weight() float64 { - var rv float64 - for _, searcher := range s.searchers { - rv += searcher.Weight() - } - return rv -} - -func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { - for _, searcher := range s.searchers { - searcher.SetQueryNorm(qnorm) - } -} - -func (s *DisjunctionSearcher) Next() (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - var err error - var rv *search.DocumentMatch - matching := make([]*search.DocumentMatch, 0, len(s.searchers)) - - found := false - for !found && s.currentID != "" { - for _, curr := range s.currs { - if curr != nil && curr.ID == s.currentID { - matching = append(matching, curr) - } - } - - if len(matching) >= int(s.min) { - found = true - // score this match - rv = s.scorer.Score(matching, len(matching), len(s.searchers)) - } - - // reset matching - matching = make([]*search.DocumentMatch, 0) - // invoke next on all the matching searchers - for i, curr := range s.currs { - if curr != nil && curr.ID == s.currentID { - searcher := s.searchers[i] - s.currs[i], err = searcher.Next() - if err != nil { - return nil, err - } - } - } - s.currentID = s.nextSmallestID() - } - return rv, nil -} - -func (s *DisjunctionSearcher) Advance(ID string) (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - // get all searchers pointing at their first match - var err error - for i, termSearcher := range s.searchers { - s.currs[i], err = termSearcher.Advance(ID) - if err != nil { - return nil, err - } - } - - s.currentID = s.nextSmallestID() - - return s.Next() -} - -func (s *DisjunctionSearcher) Count() uint64 { - // for now return a worst case - var sum uint64 - for _, searcher := range s.searchers { - sum += searcher.Count() - } - return sum -} - -func (s *DisjunctionSearcher) Close() error { - for _, searcher := range s.searchers { - err := searcher.Close() - if err != nil { - return err - } - } - return nil -} - -func (s *DisjunctionSearcher) Min() int { - return int(s.min) // FIXME just make this an int -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction_test.go deleted file mode 100644 index eee4e52a..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_disjunction_test.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestDisjunctionSearch(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - martyOrDustinSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, true) - if err != nil { - t.Fatal(err) - } - - martyTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher2, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - martyOrDustinSearcher2, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher2, dustinTermSearcher2}, 0, true) - if err != nil { - t.Fatal(err) - } - - raviTermSearcher, err := NewTermSearcher(twoDocIndexReader, "ravi", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - nestedRaviOrMartyOrDustinSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{raviTermSearcher, martyOrDustinSearcher2}, 0, true) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - searcher search.Searcher - results []*search.DocumentMatch - }{ - { - searcher: martyOrDustinSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 0.6775110856165737, - }, - &search.DocumentMatch{ - ID: "3", - Score: 0.6775110856165737, - }, - }, - }, - // test a nested disjunction - { - searcher: nestedRaviOrMartyOrDustinSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 0.2765927424732821, - }, - &search.DocumentMatch{ - ID: "3", - Score: 0.2765927424732821, - }, - &search.DocumentMatch{ - ID: "4", - Score: 0.5531854849465642, - }, - }, - }, - } - - for testIndex, test := range tests { - defer func() { - err := test.searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - next, err := test.searcher.Next() - i := 0 - for err == nil && next != nil { - if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) - } - if !scoresCloseEnough(next.Score, test.results[i].Score) { - t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) - t.Logf("scoring explanation: %s", next.Expl) - } - } - next, err = test.searcher.Next() - i++ - } - if err != nil { - t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) - } - if len(test.results) != i { - t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) - } - } -} - -func TestDisjunctionAdvance(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - martyTermSearcher, err := NewTermSearcher(twoDocIndexReader, "marty", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - dustinTermSearcher, err := NewTermSearcher(twoDocIndexReader, "dustin", "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - martyOrDustinSearcher, err := NewDisjunctionSearcher(twoDocIndexReader, []search.Searcher{martyTermSearcher, dustinTermSearcher}, 0, true) - if err != nil { - t.Fatal(err) - } - - match, err := martyOrDustinSearcher.Advance("3") - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if match == nil { - t.Errorf("expected 3, got nil") - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_fuzzy.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_fuzzy.go deleted file mode 100644 index 2cfa8329..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_fuzzy.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type FuzzySearcher struct { - indexReader index.IndexReader - term string - prefix int - fuzziness int - field string - explain bool - searcher *DisjunctionSearcher -} - -func NewFuzzySearcher(indexReader index.IndexReader, term string, prefix, fuzziness int, field string, boost float64, explain bool) (*FuzzySearcher, error) { - prefixTerm := "" - for i, r := range term { - if i < prefix { - prefixTerm += string(r) - } - } - - // find the terms with this prefix - var fieldDict index.FieldDict - var err error - if len(prefixTerm) > 0 { - fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) - } else { - fieldDict, err = indexReader.FieldDict(field) - } - - // enumerate terms and check levenshtein distance - candidateTerms := make([]string, 0) - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - ld, exceeded := search.LevenshteinDistanceMax(&term, &tfd.Term, fuzziness) - if !exceeded && ld <= fuzziness { - candidateTerms = append(candidateTerms, tfd.Term) - } - tfd, err = fieldDict.Next() - } - if err != nil { - return nil, err - } - - // enumerate all the terms in the range - qsearchers := make([]search.Searcher, 0, 25) - - for _, cterm := range candidateTerms { - qsearcher, err := NewTermSearcher(indexReader, cterm, field, 1.0, explain) - if err != nil { - return nil, err - } - qsearchers = append(qsearchers, qsearcher) - } - - // build disjunction searcher of these ranges - searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, explain) - if err != nil { - return nil, err - } - - return &FuzzySearcher{ - indexReader: indexReader, - term: term, - prefix: prefix, - fuzziness: fuzziness, - field: field, - explain: explain, - searcher: searcher, - }, nil -} -func (s *FuzzySearcher) Count() uint64 { - return s.searcher.Count() -} - -func (s *FuzzySearcher) Weight() float64 { - return s.searcher.Weight() -} - -func (s *FuzzySearcher) SetQueryNorm(qnorm float64) { - s.searcher.SetQueryNorm(qnorm) -} - -func (s *FuzzySearcher) Next() (*search.DocumentMatch, error) { - return s.searcher.Next() - -} - -func (s *FuzzySearcher) Advance(ID string) (*search.DocumentMatch, error) { - return s.searcher.Next() -} - -func (s *FuzzySearcher) Close() error { - return s.searcher.Close() -} - -func (s *FuzzySearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_all.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_match_all.go deleted file mode 100644 index 657bbaff..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_all.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorers" -) - -type MatchAllSearcher struct { - indexReader index.IndexReader - reader index.DocIDReader - scorer *scorers.ConstantScorer -} - -func NewMatchAllSearcher(indexReader index.IndexReader, boost float64, explain bool) (*MatchAllSearcher, error) { - reader, err := indexReader.DocIDReader("", "") - if err != nil { - return nil, err - } - scorer := scorers.NewConstantScorer(1.0, boost, explain) - return &MatchAllSearcher{ - indexReader: indexReader, - reader: reader, - scorer: scorer, - }, nil -} - -func (s *MatchAllSearcher) Count() uint64 { - return s.indexReader.DocCount() -} - -func (s *MatchAllSearcher) Weight() float64 { - return s.scorer.Weight() -} - -func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) { - s.scorer.SetQueryNorm(qnorm) -} - -func (s *MatchAllSearcher) Next() (*search.DocumentMatch, error) { - id, err := s.reader.Next() - if err != nil { - return nil, err - } - - if id == "" { - return nil, nil - } - - // score match - docMatch := s.scorer.Score(id) - // return doc match - return docMatch, nil - -} - -func (s *MatchAllSearcher) Advance(ID string) (*search.DocumentMatch, error) { - id, err := s.reader.Advance(ID) - if err != nil { - return nil, err - } - - if id == "" { - return nil, nil - } - - // score match - docMatch := s.scorer.Score(id) - - // return doc match - return docMatch, nil -} - -func (s *MatchAllSearcher) Close() error { - return s.reader.Close() -} - -func (s *MatchAllSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_all_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_match_all_test.go deleted file mode 100644 index c56ba601..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_all_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2013 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestMatchAllSearch(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - allSearcher, err := NewMatchAllSearcher(twoDocIndexReader, 1.0, true) - if err != nil { - t.Fatal(err) - } - - allSearcher2, err := NewMatchAllSearcher(twoDocIndexReader, 1.2, true) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - searcher search.Searcher - queryNorm float64 - results []*search.DocumentMatch - }{ - { - searcher: allSearcher, - queryNorm: 1.0, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "2", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "3", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "4", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "5", - Score: 1.0, - }, - }, - }, - { - searcher: allSearcher2, - queryNorm: 0.8333333, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "2", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "3", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "4", - Score: 1.0, - }, - &search.DocumentMatch{ - ID: "5", - Score: 1.0, - }, - }, - }, - } - - for testIndex, test := range tests { - - if test.queryNorm != 1.0 { - test.searcher.SetQueryNorm(test.queryNorm) - } - defer func() { - err := test.searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - next, err := test.searcher.Next() - i := 0 - for err == nil && next != nil { - if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) - } - if !scoresCloseEnough(next.Score, test.results[i].Score) { - t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) - t.Logf("scoring explanation: %s", next.Expl) - } - } - next, err = test.searcher.Next() - i++ - } - if err != nil { - t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) - } - if len(test.results) != i { - t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_none.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_match_none.go deleted file mode 100644 index 87881f5c..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_none.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type MatchNoneSearcher struct { - indexReader index.IndexReader -} - -func NewMatchNoneSearcher(indexReader index.IndexReader) (*MatchNoneSearcher, error) { - return &MatchNoneSearcher{ - indexReader: indexReader, - }, nil -} - -func (s *MatchNoneSearcher) Count() uint64 { - return uint64(0) -} - -func (s *MatchNoneSearcher) Weight() float64 { - return 0.0 -} - -func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) { - -} - -func (s *MatchNoneSearcher) Next() (*search.DocumentMatch, error) { - return nil, nil -} - -func (s *MatchNoneSearcher) Advance(ID string) (*search.DocumentMatch, error) { - return nil, nil -} - -func (s *MatchNoneSearcher) Close() error { - return nil -} - -func (s *MatchNoneSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_none_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_match_none_test.go deleted file mode 100644 index 8a33c921..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_match_none_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) 2013 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestMatchNoneSearch(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - noneSearcher, err := NewMatchNoneSearcher(twoDocIndexReader) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - searcher search.Searcher - results []*search.DocumentMatch - }{ - { - searcher: noneSearcher, - results: []*search.DocumentMatch{}, - }, - } - - for testIndex, test := range tests { - defer func() { - err := test.searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - next, err := test.searcher.Next() - i := 0 - for err == nil && next != nil { - if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) - } - if !scoresCloseEnough(next.Score, test.results[i].Score) { - t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) - t.Logf("scoring explanation: %s", next.Expl) - } - } - next, err = test.searcher.Next() - i++ - } - if err != nil { - t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) - } - if len(test.results) != i { - t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_numeric_range_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_numeric_range_test.go deleted file mode 100644 index f0f96342..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_numeric_range_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "reflect" - "testing" - - "github.com/blevesearch/bleve/numeric_util" -) - -func TestSplitRange(t *testing.T) { - min := numeric_util.Float64ToInt64(1.0) - max := numeric_util.Float64ToInt64(5.0) - ranges := splitInt64Range(min, max, 4) - enumerated := ranges.Enumerate() - if len(enumerated) != 135 { - t.Errorf("expected 135 terms, got %d", len(enumerated)) - } - -} - -func TestIncrementBytes(t *testing.T) { - tests := []struct { - in []byte - out []byte - }{ - { - in: []byte{0}, - out: []byte{1}, - }, - { - in: []byte{0, 0}, - out: []byte{0, 1}, - }, - { - in: []byte{0, 255}, - out: []byte{1, 0}, - }, - } - - for _, test := range tests { - actual := incrementBytes(test.in) - if !reflect.DeepEqual(actual, test.out) { - t.Errorf("expected %#v, got %#v", test.out, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_phrase.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_phrase.go deleted file mode 100644 index 8f5b4e96..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_phrase.go +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "math" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type PhraseSearcher struct { - initialized bool - indexReader index.IndexReader - mustSearcher *ConjunctionSearcher - queryNorm float64 - currMust *search.DocumentMatch - slop int - terms []string -} - -func NewPhraseSearcher(indexReader index.IndexReader, mustSearcher *ConjunctionSearcher, terms []string) (*PhraseSearcher, error) { - // build our searcher - rv := PhraseSearcher{ - indexReader: indexReader, - mustSearcher: mustSearcher, - terms: terms, - } - rv.computeQueryNorm() - return &rv, nil -} - -func (s *PhraseSearcher) computeQueryNorm() { - // first calculate sum of squared weights - sumOfSquaredWeights := 0.0 - if s.mustSearcher != nil { - sumOfSquaredWeights += s.mustSearcher.Weight() - } - - // now compute query norm from this - s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) - // finally tell all the downstream searchers the norm - if s.mustSearcher != nil { - s.mustSearcher.SetQueryNorm(s.queryNorm) - } -} - -func (s *PhraseSearcher) initSearchers() error { - var err error - // get all searchers pointing at their first match - if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next() - if err != nil { - return err - } - } - - s.initialized = true - return nil -} - -func (s *PhraseSearcher) advanceNextMust() error { - var err error - - if s.mustSearcher != nil { - s.currMust, err = s.mustSearcher.Next() - if err != nil { - return err - } - } - - return nil -} - -func (s *PhraseSearcher) Weight() float64 { - var rv float64 - rv += s.mustSearcher.Weight() - - return rv -} - -func (s *PhraseSearcher) SetQueryNorm(qnorm float64) { - s.mustSearcher.SetQueryNorm(qnorm) -} - -func (s *PhraseSearcher) Next() (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - - var rv *search.DocumentMatch - for s.currMust != nil { - rvftlm := make(search.FieldTermLocationMap, 0) - freq := 0 - firstTerm := s.terms[0] - for field, termLocMap := range s.currMust.Locations { - rvtlm := make(search.TermLocationMap, 0) - locations, ok := termLocMap[firstTerm] - if ok { - OUTER: - for _, location := range locations { - crvtlm := make(search.TermLocationMap, 0) - INNER: - for i := 0; i < len(s.terms); i++ { - nextTerm := s.terms[i] - if nextTerm != "" { - // look through all these term locations - // to try and find the correct offsets - nextLocations, ok := termLocMap[nextTerm] - if ok { - for _, nextLocation := range nextLocations { - if nextLocation.Pos == location.Pos+float64(i) { - // found a location match for this term - crvtlm.AddLocation(nextTerm, nextLocation) - continue INNER - } - } - // if we got here we didn't find a location match for this term - continue OUTER - } else { - continue OUTER - } - } - } - // if we got here all the terms matched - freq++ - search.MergeTermLocationMaps(rvtlm, crvtlm) - rvftlm[field] = rvtlm - } - } - } - - if freq > 0 { - // return match - rv = s.currMust - rv.Locations = rvftlm - err := s.advanceNextMust() - if err != nil { - return nil, err - } - return rv, nil - } - - err := s.advanceNextMust() - if err != nil { - return nil, err - } - } - - return nil, nil -} - -func (s *PhraseSearcher) Advance(ID string) (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers() - if err != nil { - return nil, err - } - } - var err error - s.currMust, err = s.mustSearcher.Advance(ID) - if err != nil { - return nil, err - } - return s.Next() -} - -func (s *PhraseSearcher) Count() uint64 { - // for now return a worst case - var sum uint64 - sum += s.mustSearcher.Count() - return sum -} - -func (s *PhraseSearcher) Close() error { - if s.mustSearcher != nil { - err := s.mustSearcher.Close() - if err != nil { - return err - } - } - return nil -} - -func (s *PhraseSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_phrase_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_phrase_test.go deleted file mode 100644 index ceda410c..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_phrase_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) 2013 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestPhraseSearch(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - angstTermSearcher, err := NewTermSearcher(twoDocIndexReader, "angst", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - beerTermSearcher, err := NewTermSearcher(twoDocIndexReader, "beer", "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - mustSearcher, err := NewConjunctionSearcher(twoDocIndexReader, []search.Searcher{angstTermSearcher, beerTermSearcher}, true) - if err != nil { - t.Fatal(err) - } - phraseSearcher, err := NewPhraseSearcher(twoDocIndexReader, mustSearcher, []string{"angst", "beer"}) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - searcher search.Searcher - results []*search.DocumentMatch - }{ - { - searcher: phraseSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "2", - Score: 1.0807601687084403, - }, - }, - }, - } - - for testIndex, test := range tests { - defer func() { - err := test.searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - next, err := test.searcher.Next() - i := 0 - for err == nil && next != nil { - if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) - } - if next.Score != test.results[i].Score { - t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) - t.Logf("scoring explanation: %s", next.Expl) - } - } - next, err = test.searcher.Next() - i++ - } - if err != nil { - t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) - } - if len(test.results) != i { - t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_regexp.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_regexp.go deleted file mode 100644 index e4cc5c24..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_regexp.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "regexp" - - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type RegexpSearcher struct { - indexReader index.IndexReader - pattern *regexp.Regexp - field string - explain bool - searcher *DisjunctionSearcher -} - -func NewRegexpSearcher(indexReader index.IndexReader, pattern *regexp.Regexp, field string, boost float64, explain bool) (*RegexpSearcher, error) { - - prefixTerm, complete := pattern.LiteralPrefix() - candidateTerms := make([]string, 0) - if complete { - // there is no pattern - candidateTerms = append(candidateTerms, prefixTerm) - } else { - var fieldDict index.FieldDict - var err error - if len(prefixTerm) > 0 { - fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) - } else { - fieldDict, err = indexReader.FieldDict(field) - } - - // enumerate the terms and check against regexp - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - if pattern.MatchString(tfd.Term) { - candidateTerms = append(candidateTerms, tfd.Term) - } - tfd, err = fieldDict.Next() - } - if err != nil { - return nil, err - } - } - - // enumerate all the terms in the range - qsearchers := make([]search.Searcher, 0, 25) - - for _, cterm := range candidateTerms { - qsearcher, err := NewTermSearcher(indexReader, cterm, field, 1.0, explain) - if err != nil { - return nil, err - } - qsearchers = append(qsearchers, qsearcher) - } - - // build disjunction searcher of these ranges - searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, explain) - if err != nil { - return nil, err - } - - return &RegexpSearcher{ - indexReader: indexReader, - pattern: pattern, - field: field, - explain: explain, - searcher: searcher, - }, nil -} -func (s *RegexpSearcher) Count() uint64 { - return s.searcher.Count() -} - -func (s *RegexpSearcher) Weight() float64 { - return s.searcher.Weight() -} - -func (s *RegexpSearcher) SetQueryNorm(qnorm float64) { - s.searcher.SetQueryNorm(qnorm) -} - -func (s *RegexpSearcher) Next() (*search.DocumentMatch, error) { - return s.searcher.Next() - -} - -func (s *RegexpSearcher) Advance(ID string) (*search.DocumentMatch, error) { - return s.searcher.Next() -} - -func (s *RegexpSearcher) Close() error { - return s.searcher.Close() -} - -func (s *RegexpSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_regexp_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_regexp_test.go deleted file mode 100644 index 19534784..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_regexp_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2015 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "regexp" - "testing" - - "github.com/blevesearch/bleve/search" -) - -func TestRegexpSearch(t *testing.T) { - - twoDocIndexReader, err := twoDocIndex.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := twoDocIndexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - pattern, err := regexp.Compile("ma.*") - if err != nil { - t.Fatal(err) - } - - regexpSearcher, err := NewRegexpSearcher(twoDocIndexReader, pattern, "name", 1.0, true) - if err != nil { - t.Fatal(err) - } - - patternCo, err := regexp.Compile("co.*") - if err != nil { - t.Fatal(err) - } - - regexpSearcherCo, err := NewRegexpSearcher(twoDocIndexReader, patternCo, "desc", 1.0, true) - if err != nil { - t.Fatal(err) - } - - tests := []struct { - searcher search.Searcher - results []*search.DocumentMatch - }{ - { - searcher: regexpSearcher, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "1", - Score: 1.916290731874155, - }, - }, - }, - { - searcher: regexpSearcherCo, - results: []*search.DocumentMatch{ - &search.DocumentMatch{ - ID: "2", - Score: 0.33875554280828685, - }, - &search.DocumentMatch{ - ID: "3", - Score: 0.33875554280828685, - }, - }, - }, - } - - for testIndex, test := range tests { - defer func() { - err := test.searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - next, err := test.searcher.Next() - i := 0 - for err == nil && next != nil { - if i < len(test.results) { - if next.ID != test.results[i].ID { - t.Errorf("expected result %d to have id %s got %s for test %d", i, test.results[i].ID, next.ID, testIndex) - } - if next.Score != test.results[i].Score { - t.Errorf("expected result %d to have score %v got %v for test %d", i, test.results[i].Score, next.Score, testIndex) - t.Logf("scoring explanation: %s", next.Expl) - } - } - next, err = test.searcher.Next() - i++ - } - if err != nil { - t.Fatalf("error iterating searcher: %v for test %d", err, testIndex) - } - if len(test.results) != i { - t.Errorf("expected %d results got %d for test %d", len(test.results), i, testIndex) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_term.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_term.go deleted file mode 100644 index 2ea3a9d8..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_term.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorers" -) - -type TermSearcher struct { - indexReader index.IndexReader - term string - field string - explain bool - reader index.TermFieldReader - scorer *scorers.TermQueryScorer -} - -func NewTermSearcher(indexReader index.IndexReader, term string, field string, boost float64, explain bool) (*TermSearcher, error) { - reader, err := indexReader.TermFieldReader([]byte(term), field) - if err != nil { - return nil, err - } - scorer := scorers.NewTermQueryScorer(term, field, boost, indexReader.DocCount(), reader.Count(), explain) - return &TermSearcher{ - indexReader: indexReader, - term: term, - field: field, - explain: explain, - reader: reader, - scorer: scorer, - }, nil -} - -func (s *TermSearcher) Count() uint64 { - return s.reader.Count() -} - -func (s *TermSearcher) Weight() float64 { - return s.scorer.Weight() -} - -func (s *TermSearcher) SetQueryNorm(qnorm float64) { - s.scorer.SetQueryNorm(qnorm) -} - -func (s *TermSearcher) Next() (*search.DocumentMatch, error) { - termMatch, err := s.reader.Next() - if err != nil { - return nil, err - } - - if termMatch == nil { - return nil, nil - } - - // score match - docMatch := s.scorer.Score(termMatch) - // return doc match - return docMatch, nil - -} - -func (s *TermSearcher) Advance(ID string) (*search.DocumentMatch, error) { - termMatch, err := s.reader.Advance(ID) - if err != nil { - return nil, err - } - - if termMatch == nil { - return nil, nil - } - - // score match - docMatch := s.scorer.Score(termMatch) - - // return doc match - return docMatch, nil -} - -func (s *TermSearcher) Close() error { - return s.reader.Close() -} - -func (s *TermSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_term_prefix.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_term_prefix.go deleted file mode 100644 index d2d562b0..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_term_prefix.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "github.com/blevesearch/bleve/index" - "github.com/blevesearch/bleve/search" -) - -type TermPrefixSearcher struct { - indexReader index.IndexReader - prefix string - field string - explain bool - searcher *DisjunctionSearcher -} - -func NewTermPrefixSearcher(indexReader index.IndexReader, prefix string, field string, boost float64, explain bool) (*TermPrefixSearcher, error) { - // find the terms with this prefix - fieldDict, err := indexReader.FieldDictPrefix(field, []byte(prefix)) - - // enumerate all the terms in the range - qsearchers := make([]search.Searcher, 0, 25) - tfd, err := fieldDict.Next() - for err == nil && tfd != nil { - qsearcher, err := NewTermSearcher(indexReader, string(tfd.Term), field, 1.0, explain) - if err != nil { - return nil, err - } - qsearchers = append(qsearchers, qsearcher) - tfd, err = fieldDict.Next() - } - // build disjunction searcher of these ranges - searcher, err := NewDisjunctionSearcher(indexReader, qsearchers, 0, explain) - if err != nil { - return nil, err - } - - return &TermPrefixSearcher{ - indexReader: indexReader, - prefix: prefix, - field: field, - explain: explain, - searcher: searcher, - }, nil -} -func (s *TermPrefixSearcher) Count() uint64 { - return s.searcher.Count() -} - -func (s *TermPrefixSearcher) Weight() float64 { - return s.searcher.Weight() -} - -func (s *TermPrefixSearcher) SetQueryNorm(qnorm float64) { - s.searcher.SetQueryNorm(qnorm) -} - -func (s *TermPrefixSearcher) Next() (*search.DocumentMatch, error) { - return s.searcher.Next() - -} - -func (s *TermPrefixSearcher) Advance(ID string) (*search.DocumentMatch, error) { - return s.searcher.Next() -} - -func (s *TermPrefixSearcher) Close() error { - return s.searcher.Close() -} - -func (s *TermPrefixSearcher) Min() int { - return 0 -} diff --git a/vendor/github.com/blevesearch/bleve/search/searchers/search_term_test.go b/vendor/github.com/blevesearch/bleve/search/searchers/search_term_test.go deleted file mode 100644 index e5b200bf..00000000 --- a/vendor/github.com/blevesearch/bleve/search/searchers/search_term_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright (c) 2013 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package searchers - -import ( - "math" - "testing" - - "github.com/blevesearch/bleve/document" - "github.com/blevesearch/bleve/index/store/inmem" - "github.com/blevesearch/bleve/index/upside_down" -) - -func TestTermSearcher(t *testing.T) { - - var queryTerm = "beer" - var queryField = "desc" - var queryBoost = 3.0 - var queryExplain = true - - inMemStore, _ := inmem.New() - analysisQueue := upside_down.NewAnalysisQueue(1) - i := upside_down.NewUpsideDownCouch(inMemStore, analysisQueue) - err := i.Open() - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "a", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "b", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "c", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "d", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "e", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "f", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "g", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "h", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "i", - Fields: []document.Field{ - document.NewTextField("desc", []uint64{}, []byte("beer")), - }, - }) - if err != nil { - t.Fatal(err) - } - err = i.Update(&document.Document{ - ID: "j", - Fields: []document.Field{ - document.NewTextField("title", []uint64{}, []byte("cat")), - }, - }) - if err != nil { - t.Fatal(err) - } - - indexReader, err := i.Reader() - if err != nil { - t.Error(err) - } - defer func() { - err := indexReader.Close() - if err != nil { - t.Fatal(err) - } - }() - - searcher, err := NewTermSearcher(indexReader, queryTerm, queryField, queryBoost, queryExplain) - if err != nil { - t.Fatal(err) - } - defer func() { - err := searcher.Close() - if err != nil { - t.Fatal(err) - } - }() - - searcher.SetQueryNorm(2.0) - docCount, err := i.DocCount() - if err != nil { - t.Fatal(err) - } - idf := 1.0 + math.Log(float64(docCount)/float64(searcher.Count()+1.0)) - expectedQueryWeight := 3 * idf * 3 * idf - if expectedQueryWeight != searcher.Weight() { - t.Errorf("expected weight %v got %v", expectedQueryWeight, searcher.Weight()) - } - - if searcher.Count() != 9 { - t.Errorf("expected count of 9, got %d", searcher.Count()) - } - - docMatch, err := searcher.Next() - if err != nil { - t.Errorf("expected result, got %v", err) - } - if docMatch.ID != "a" { - t.Errorf("expected result ID to be 'a', got '%s", docMatch.ID) - } - docMatch, err = searcher.Advance("c") - if err != nil { - t.Errorf("expected result, got %v", err) - } - if docMatch.ID != "c" { - t.Errorf("expected result ID to be 'c' got '%s'", docMatch.ID) - } - - // try advancing past end - docMatch, err = searcher.Advance("z") - if err != nil { - t.Fatal(err) - } - if docMatch != nil { - t.Errorf("expected nil, got %v", docMatch) - } - - // try pushing next past end - docMatch, err = searcher.Next() - if err != nil { - t.Fatal(err) - } - if docMatch != nil { - t.Errorf("expected nil, got %v", docMatch) - } -} diff --git a/vendor/github.com/blevesearch/bleve/search/sort.go b/vendor/github.com/blevesearch/bleve/search/sort.go new file mode 100644 index 00000000..1987718b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/sort.go @@ -0,0 +1,711 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package search + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strings" + + "github.com/blevesearch/bleve/geo" + "github.com/blevesearch/bleve/numeric" +) + +var HighTerm = strings.Repeat(string([]byte{0xff}), 10) +var LowTerm = string([]byte{0x00}) + +type SearchSort interface { + UpdateVisitor(field string, term []byte) + Value(a *DocumentMatch) string + Descending() bool + + RequiresDocID() bool + RequiresScoring() bool + RequiresFields() []string + + Copy() SearchSort +} + +func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { + descending, ok := input["desc"].(bool) + by, ok := input["by"].(string) + if !ok { + return nil, fmt.Errorf("search sort must specify by") + } + switch by { + case "id": + return &SortDocID{ + Desc: descending, + }, nil + case "score": + return &SortScore{ + Desc: descending, + }, nil + case "geo_distance": + field, ok := input["field"].(string) + if !ok { + return nil, fmt.Errorf("search sort mode geo_distance must specify field") + } + lon, lat, foundLocation := geo.ExtractGeoPoint(input["location"]) + if !foundLocation { + return nil, fmt.Errorf("unable to parse geo_distance location") + } + rvd := &SortGeoDistance{ + Field: field, + Desc: descending, + lon: lon, + lat: lat, + unitMult: 1.0, + } + if distUnit, ok := input["unit"].(string); ok { + var err error + rvd.unitMult, err = geo.ParseDistanceUnit(distUnit) + if err != nil { + return nil, err + } + rvd.Unit = distUnit + } + return rvd, nil + case "field": + field, ok := input["field"].(string) + if !ok { + return nil, fmt.Errorf("search sort mode field must specify field") + } + rv := &SortField{ + Field: field, + Desc: descending, + } + typ, ok := input["type"].(string) + if ok { + switch typ { + case "auto": + rv.Type = SortFieldAuto + case "string": + rv.Type = SortFieldAsString + case "number": + rv.Type = SortFieldAsNumber + case "date": + rv.Type = SortFieldAsDate + default: + return nil, fmt.Errorf("unknown sort field type: %s", typ) + } + } + mode, ok := input["mode"].(string) + if ok { + switch mode { + case "default": + rv.Mode = SortFieldDefault + case "min": + rv.Mode = SortFieldMin + case "max": + rv.Mode = SortFieldMax + default: + return nil, fmt.Errorf("unknown sort field mode: %s", mode) + } + } + missing, ok := input["missing"].(string) + if ok { + switch missing { + case "first": + rv.Missing = SortFieldMissingFirst + case "last": + rv.Missing = SortFieldMissingLast + default: + return nil, fmt.Errorf("unknown sort field missing: %s", missing) + } + } + return rv, nil + } + + return nil, fmt.Errorf("unknown search sort by: %s", by) +} + +func ParseSearchSortString(input string) SearchSort { + descending := false + if strings.HasPrefix(input, "-") { + descending = true + input = input[1:] + } else if strings.HasPrefix(input, "+") { + input = input[1:] + } + if input == "_id" { + return &SortDocID{ + Desc: descending, + } + } else if input == "_score" { + return &SortScore{ + Desc: descending, + } + } + return &SortField{ + Field: input, + Desc: descending, + } +} + +func ParseSearchSortJSON(input json.RawMessage) (SearchSort, error) { + // first try to parse it as string + var sortString string + err := json.Unmarshal(input, &sortString) + if err != nil { + var sortObj map[string]interface{} + err = json.Unmarshal(input, &sortObj) + if err != nil { + return nil, err + } + return ParseSearchSortObj(sortObj) + } + return ParseSearchSortString(sortString), nil +} + +func ParseSortOrderStrings(in []string) SortOrder { + rv := make(SortOrder, 0, len(in)) + for _, i := range in { + ss := ParseSearchSortString(i) + rv = append(rv, ss) + } + return rv +} + +func ParseSortOrderJSON(in []json.RawMessage) (SortOrder, error) { + rv := make(SortOrder, 0, len(in)) + for _, i := range in { + ss, err := ParseSearchSortJSON(i) + if err != nil { + return nil, err + } + rv = append(rv, ss) + } + return rv, nil +} + +type SortOrder []SearchSort + +func (so SortOrder) Value(doc *DocumentMatch) { + for _, soi := range so { + doc.Sort = append(doc.Sort, soi.Value(doc)) + } +} + +func (so SortOrder) UpdateVisitor(field string, term []byte) { + for _, soi := range so { + soi.UpdateVisitor(field, term) + } +} + +func (so SortOrder) Copy() SortOrder { + rv := make(SortOrder, len(so)) + for i, soi := range so { + rv[i] = soi.Copy() + } + return rv +} + +// Compare will compare two document matches using the specified sort order +// if both are numbers, we avoid converting back to term +func (so SortOrder) Compare(cachedScoring, cachedDesc []bool, i, j *DocumentMatch) int { + // compare the documents on all search sorts until a differences is found + for x := range so { + c := 0 + if cachedScoring[x] { + if i.Score < j.Score { + c = -1 + } else if i.Score > j.Score { + c = 1 + } + } else { + iVal := i.Sort[x] + jVal := j.Sort[x] + c = strings.Compare(iVal, jVal) + } + + if c == 0 { + continue + } + if cachedDesc[x] { + c = -c + } + return c + } + // if they are the same at this point, impose order based on index natural sort order + if i.HitNumber == j.HitNumber { + return 0 + } else if i.HitNumber > j.HitNumber { + return 1 + } + return -1 +} + +func (so SortOrder) RequiresScore() bool { + rv := false + for _, soi := range so { + if soi.RequiresScoring() { + rv = true + } + } + return rv +} + +func (so SortOrder) RequiresDocID() bool { + rv := false + for _, soi := range so { + if soi.RequiresDocID() { + rv = true + } + } + return rv +} + +func (so SortOrder) RequiredFields() []string { + var rv []string + for _, soi := range so { + rv = append(rv, soi.RequiresFields()...) + } + return rv +} + +func (so SortOrder) CacheIsScore() []bool { + var rv []bool + for _, soi := range so { + rv = append(rv, soi.RequiresScoring()) + } + return rv +} + +func (so SortOrder) CacheDescending() []bool { + var rv []bool + for _, soi := range so { + rv = append(rv, soi.Descending()) + } + return rv +} + +// SortFieldType lets you control some internal sort behavior +// normally leaving this to the zero-value of SortFieldAuto is fine +type SortFieldType int + +const ( + // SortFieldAuto applies heuristics attempt to automatically sort correctly + SortFieldAuto SortFieldType = iota + // SortFieldAsString forces sort as string (no prefix coded terms removed) + SortFieldAsString + // SortFieldAsNumber forces sort as string (prefix coded terms with shift > 0 removed) + SortFieldAsNumber + // SortFieldAsDate forces sort as string (prefix coded terms with shift > 0 removed) + SortFieldAsDate +) + +// SortFieldMode describes the behavior if the field has multiple values +type SortFieldMode int + +const ( + // SortFieldDefault uses the first (or only) value, this is the default zero-value + SortFieldDefault SortFieldMode = iota // FIXME name is confusing + // SortFieldMin uses the minimum value + SortFieldMin + // SortFieldMax uses the maximum value + SortFieldMax +) + +// SortFieldMissing controls where documents missing a field value should be sorted +type SortFieldMissing int + +const ( + // SortFieldMissingLast sorts documents missing a field at the end + SortFieldMissingLast SortFieldMissing = iota + + // SortFieldMissingFirst sorts documents missing a field at the beginning + SortFieldMissingFirst +) + +// SortField will sort results by the value of a stored field +// Field is the name of the field +// Descending reverse the sort order (default false) +// Type allows forcing of string/number/date behavior (default auto) +// Mode controls behavior for multi-values fields (default first) +// Missing controls behavior of missing values (default last) +type SortField struct { + Field string + Desc bool + Type SortFieldType + Mode SortFieldMode + Missing SortFieldMissing + values []string +} + +// UpdateVisitor notifies this sort field that in this document +// this field has the specified term +func (s *SortField) UpdateVisitor(field string, term []byte) { + if field == s.Field { + s.values = append(s.values, string(term)) + } +} + +// Value returns the sort value of the DocumentMatch +// it also resets the state of this SortField for +// processing the next document +func (s *SortField) Value(i *DocumentMatch) string { + iTerms := s.filterTermsByType(s.values) + iTerm := s.filterTermsByMode(iTerms) + s.values = nil + return iTerm +} + +// Descending determines the order of the sort +func (s *SortField) Descending() bool { + return s.Desc +} + +func (s *SortField) filterTermsByMode(terms []string) string { + if len(terms) == 1 || (len(terms) > 1 && s.Mode == SortFieldDefault) { + return terms[0] + } else if len(terms) > 1 { + switch s.Mode { + case SortFieldMin: + sort.Strings(terms) + return terms[0] + case SortFieldMax: + sort.Strings(terms) + return terms[len(terms)-1] + } + } + + // handle missing terms + if s.Missing == SortFieldMissingLast { + if s.Desc { + return LowTerm + } + return HighTerm + } + if s.Desc { + return HighTerm + } + return LowTerm +} + +// filterTermsByType attempts to make one pass on the terms +// if we are in auto-mode AND all the terms look like prefix-coded numbers +// return only the terms which had shift of 0 +// if we are in explicit number or date mode, return only valid +// prefix coded numbers with shift of 0 +func (s *SortField) filterTermsByType(terms []string) []string { + stype := s.Type + if stype == SortFieldAuto { + allTermsPrefixCoded := true + var termsWithShiftZero []string + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTerm(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } else if !valid { + allTermsPrefixCoded = false + } + } + if allTermsPrefixCoded { + terms = termsWithShiftZero + } + } else if stype == SortFieldAsNumber || stype == SortFieldAsDate { + var termsWithShiftZero []string + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTerm(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } + } + terms = termsWithShiftZero + } + return terms +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortField) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortField) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore requires the specified stored field +func (s *SortField) RequiresFields() []string { return []string{s.Field} } + +func (s *SortField) MarshalJSON() ([]byte, error) { + // see if simple format can be used + if s.Missing == SortFieldMissingLast && + s.Mode == SortFieldDefault && + s.Type == SortFieldAuto { + if s.Desc { + return json.Marshal("-" + s.Field) + } + return json.Marshal(s.Field) + } + sfm := map[string]interface{}{ + "by": "field", + "field": s.Field, + } + if s.Desc { + sfm["desc"] = true + } + if s.Missing > SortFieldMissingLast { + switch s.Missing { + case SortFieldMissingFirst: + sfm["missing"] = "first" + } + } + if s.Mode > SortFieldDefault { + switch s.Mode { + case SortFieldMin: + sfm["mode"] = "min" + case SortFieldMax: + sfm["mode"] = "max" + } + } + if s.Type > SortFieldAuto { + switch s.Type { + case SortFieldAsString: + sfm["type"] = "string" + case SortFieldAsNumber: + sfm["type"] = "number" + case SortFieldAsDate: + sfm["type"] = "date" + } + } + + return json.Marshal(sfm) +} + +func (s *SortField) Copy() SearchSort { + var rv SortField + rv = *s + return &rv +} + +// SortDocID will sort results by the document identifier +type SortDocID struct { + Desc bool +} + +// UpdateVisitor is a no-op for SortDocID as it's value +// is not dependent on any field terms +func (s *SortDocID) UpdateVisitor(field string, term []byte) { + +} + +// Value returns the sort value of the DocumentMatch +func (s *SortDocID) Value(i *DocumentMatch) string { + return i.ID +} + +// Descending determines the order of the sort +func (s *SortDocID) Descending() bool { + return s.Desc +} + +// RequiresDocID says this SearchSort does require the DocID be loaded +func (s *SortDocID) RequiresDocID() bool { return true } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortDocID) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore does not require any stored fields +func (s *SortDocID) RequiresFields() []string { return nil } + +func (s *SortDocID) MarshalJSON() ([]byte, error) { + if s.Desc { + return json.Marshal("-_id") + } + return json.Marshal("_id") +} + +func (s *SortDocID) Copy() SearchSort { + var rv SortDocID + rv = *s + return &rv +} + +// SortScore will sort results by the document match score +type SortScore struct { + Desc bool +} + +// UpdateVisitor is a no-op for SortScore as it's value +// is not dependent on any field terms +func (s *SortScore) UpdateVisitor(field string, term []byte) { + +} + +// Value returns the sort value of the DocumentMatch +func (s *SortScore) Value(i *DocumentMatch) string { + return "_score" +} + +// Descending determines the order of the sort +func (s *SortScore) Descending() bool { + return s.Desc +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortScore) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does require scoring +func (s *SortScore) RequiresScoring() bool { return true } + +// RequiresFields says this SearchStore does not require any store fields +func (s *SortScore) RequiresFields() []string { return nil } + +func (s *SortScore) MarshalJSON() ([]byte, error) { + if s.Desc { + return json.Marshal("-_score") + } + return json.Marshal("_score") +} + +func (s *SortScore) Copy() SearchSort { + var rv SortScore + rv = *s + return &rv +} + +var maxDistance = string(numeric.MustNewPrefixCodedInt64(math.MaxInt64, 0)) + +// NewSortGeoDistance creates SearchSort instance for sorting documents by +// their distance from the specified point. +func NewSortGeoDistance(field, unit string, lon, lat float64, desc bool) ( + *SortGeoDistance, error) { + + rv := &SortGeoDistance{ + Field: field, + Desc: desc, + Unit: unit, + lon: lon, + lat: lat, + } + var err error + rv.unitMult, err = geo.ParseDistanceUnit(unit) + if err != nil { + return nil, err + } + return rv, nil +} + +// SortGeoDistance will sort results by the distance of an +// indexed geo point, from the provided location. +// Field is the name of the field +// Descending reverse the sort order (default false) +type SortGeoDistance struct { + Field string + Desc bool + Unit string + values []string + lon float64 + lat float64 + unitMult float64 +} + +// UpdateVisitor notifies this sort field that in this document +// this field has the specified term +func (s *SortGeoDistance) UpdateVisitor(field string, term []byte) { + if field == s.Field { + s.values = append(s.values, string(term)) + } +} + +// Value returns the sort value of the DocumentMatch +// it also resets the state of this SortField for +// processing the next document +func (s *SortGeoDistance) Value(i *DocumentMatch) string { + iTerms := s.filterTermsByType(s.values) + iTerm := s.filterTermsByMode(iTerms) + s.values = nil + + if iTerm == "" { + return maxDistance + } + + i64, err := numeric.PrefixCoded(iTerm).Int64() + if err != nil { + return maxDistance + } + docLon := geo.MortonUnhashLon(uint64(i64)) + docLat := geo.MortonUnhashLat(uint64(i64)) + + dist := geo.Haversin(s.lon, s.lat, docLon, docLat) + // dist is returned in km, so convert to m + dist *= 1000 + if s.unitMult != 0 { + dist /= s.unitMult + } + distInt64 := numeric.Float64ToInt64(dist) + return string(numeric.MustNewPrefixCodedInt64(distInt64, 0)) +} + +// Descending determines the order of the sort +func (s *SortGeoDistance) Descending() bool { + return s.Desc +} + +func (s *SortGeoDistance) filterTermsByMode(terms []string) string { + if len(terms) >= 1 { + return terms[0] + } + + return "" +} + +// filterTermsByType attempts to make one pass on the terms +// return only valid prefix coded numbers with shift of 0 +func (s *SortGeoDistance) filterTermsByType(terms []string) []string { + var termsWithShiftZero []string + for _, term := range terms { + valid, shift := numeric.ValidPrefixCodedTerm(term) + if valid && shift == 0 { + termsWithShiftZero = append(termsWithShiftZero, term) + } + } + return termsWithShiftZero +} + +// RequiresDocID says this SearchSort does not require the DocID be loaded +func (s *SortGeoDistance) RequiresDocID() bool { return false } + +// RequiresScoring says this SearchStore does not require scoring +func (s *SortGeoDistance) RequiresScoring() bool { return false } + +// RequiresFields says this SearchStore requires the specified stored field +func (s *SortGeoDistance) RequiresFields() []string { return []string{s.Field} } + +func (s *SortGeoDistance) MarshalJSON() ([]byte, error) { + sfm := map[string]interface{}{ + "by": "geo_distance", + "field": s.Field, + "location": map[string]interface{}{ + "lon": s.lon, + "lat": s.lat, + }, + } + if s.Unit != "" { + sfm["unit"] = s.Unit + } + if s.Desc { + sfm["desc"] = true + } + + return json.Marshal(sfm) +} + +func (s *SortGeoDistance) Copy() SearchSort { + var rv SortGeoDistance + rv = *s + return &rv +} diff --git a/vendor/github.com/blevesearch/bleve/search/util.go b/vendor/github.com/blevesearch/bleve/search/util.go index 6867eee7..83212af1 100644 --- a/vendor/github.com/blevesearch/bleve/search/util.go +++ b/vendor/github.com/blevesearch/bleve/search/util.go @@ -1,11 +1,16 @@ // Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package search diff --git a/vendor/github.com/blevesearch/bleve/search/util_test.go b/vendor/github.com/blevesearch/bleve/search/util_test.go deleted file mode 100644 index 4185646e..00000000 --- a/vendor/github.com/blevesearch/bleve/search/util_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2013 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package search - -import ( - "reflect" - "testing" -) - -func TestMergeLocations(t *testing.T) { - flm1 := FieldTermLocationMap{ - "marty": TermLocationMap{ - "name": { - &Location{ - Pos: 1, - Start: 0, - End: 5, - }, - }, - }, - } - - flm2 := FieldTermLocationMap{ - "marty": TermLocationMap{ - "description": { - &Location{ - Pos: 5, - Start: 20, - End: 25, - }, - }, - }, - } - - flm3 := FieldTermLocationMap{ - "josh": TermLocationMap{ - "description": { - &Location{ - Pos: 5, - Start: 20, - End: 25, - }, - }, - }, - } - - expectedMerge := FieldTermLocationMap{ - "marty": TermLocationMap{ - "description": { - &Location{ - Pos: 5, - Start: 20, - End: 25, - }, - }, - "name": { - &Location{ - Pos: 1, - Start: 0, - End: 5, - }, - }, - }, - "josh": TermLocationMap{ - "description": { - &Location{ - Pos: 5, - Start: 20, - End: 25, - }, - }, - }, - } - - mergedLocations := MergeLocations([]FieldTermLocationMap{flm1, flm2, flm3}) - if !reflect.DeepEqual(expectedMerge, mergedLocations) { - t.Errorf("expected %v, got %v", expectedMerge, mergedLocations) - } -} diff --git a/vendor/github.com/blevesearch/bleve/search_test.go b/vendor/github.com/blevesearch/bleve/search_test.go deleted file mode 100644 index f26a1745..00000000 --- a/vendor/github.com/blevesearch/bleve/search_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package bleve - -import ( - "reflect" - "strings" - "testing" - "time" - - "github.com/blevesearch/bleve/search" -) - -func TestSearchResultString(t *testing.T) { - - tests := []struct { - result *SearchResult - str string - }{ - { - result: &SearchResult{ - Request: &SearchRequest{ - Size: 10, - }, - Total: 5, - Took: 1 * time.Second, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{}, - &search.DocumentMatch{}, - &search.DocumentMatch{}, - &search.DocumentMatch{}, - &search.DocumentMatch{}, - }, - }, - str: "5 matches, showing 1 through 5, took 1s", - }, - { - result: &SearchResult{ - Request: &SearchRequest{ - Size: 0, - }, - Total: 5, - Hits: search.DocumentMatchCollection{}, - }, - str: "5 matches", - }, - { - result: &SearchResult{ - Request: &SearchRequest{ - Size: 10, - }, - Total: 0, - Hits: search.DocumentMatchCollection{}, - }, - str: "No matches", - }, - } - - for _, test := range tests { - srstring := test.result.String() - if !strings.HasPrefix(srstring, test.str) { - t.Errorf("expected to start %s, got %s", test.str, srstring) - } - } -} - -func TestSearchResultMerge(t *testing.T) { - l := &SearchResult{ - Total: 1, - MaxScore: 1, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", - Score: 1, - }, - }, - } - - r := &SearchResult{ - Total: 1, - MaxScore: 2, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "b", - Score: 2, - }, - }, - } - - expected := &SearchResult{ - Total: 2, - MaxScore: 2, - Hits: search.DocumentMatchCollection{ - &search.DocumentMatch{ - ID: "a", - Score: 1, - }, - &search.DocumentMatch{ - ID: "b", - Score: 2, - }, - }, - } - - l.Merge(r) - - if !reflect.DeepEqual(l, expected) { - t.Errorf("expected %#v, got %#v", expected, l) - } -} diff --git a/vendor/github.com/blevesearch/bleve/test/integration.go b/vendor/github.com/blevesearch/bleve/test/integration.go deleted file mode 100644 index da661454..00000000 --- a/vendor/github.com/blevesearch/bleve/test/integration.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package test - -import ( - "github.com/blevesearch/bleve" -) - -type SearchTest struct { - Search *bleve.SearchRequest `json:"search"` - Result *bleve.SearchResult `json:"result"` - Comment string `json:"comment"` -} - -type SearchTests []*SearchTest diff --git a/vendor/github.com/blevesearch/bleve/test/integration_test.go b/vendor/github.com/blevesearch/bleve/test/integration_test.go deleted file mode 100644 index b40daa97..00000000 --- a/vendor/github.com/blevesearch/bleve/test/integration_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package test - -import ( - "encoding/json" - "flag" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "regexp" - "testing" - - "github.com/blevesearch/bleve" -) - -var dataset = flag.String("dataset", "", "only test datasets matching this regex") -var keepIndex = flag.Bool("keepIndex", false, "keep the index after testing") - -func TestIntegration(t *testing.T) { - - flag.Parse() - - var err error - var datasetRegexp *regexp.Regexp - if *dataset != "" { - datasetRegexp, err = regexp.Compile(*dataset) - if err != nil { - t.Fatal(err) - } - } - - fis, err := ioutil.ReadDir("tests") - if err != nil { - t.Fatal(err) - } - for _, fi := range fis { - if datasetRegexp != nil { - if !datasetRegexp.MatchString(fi.Name()) { - continue - } - } - if fi.IsDir() { - t.Logf("Running test: %s", fi.Name()) - runTestDir(t, "tests"+string(filepath.Separator)+fi.Name()) - } - } -} - -func runTestDir(t *testing.T, dir string) { - // read the mapping - mappingBytes, err := ioutil.ReadFile(dir + string(filepath.Separator) + "mapping.json") - if err != nil { - t.Errorf("error reading mapping: %v", err) - return - } - var mapping bleve.IndexMapping - err = json.Unmarshal(mappingBytes, &mapping) - if err != nil { - t.Errorf("error unmarshalling mapping: %v", err) - return - } - - // open new index - if !*keepIndex { - defer func() { - err := os.RemoveAll("test.bleve") - if err != nil { - t.Fatal(err) - } - }() - } - index, err := bleve.New("test.bleve", &mapping) - if err != nil { - t.Errorf("error creating new index: %v", err) - return - } - defer func() { - err := index.Close() - if err != nil { - t.Fatal(err) - } - }() - - // index data - fis, err := ioutil.ReadDir(dir + string(filepath.Separator) + "data") - if err != nil { - t.Errorf("error reading data dir: %v", err) - return - } - for _, fi := range fis { - fileBytes, err := ioutil.ReadFile(dir + string(filepath.Separator) + "data" + string(filepath.Separator) + fi.Name()) - if err != nil { - t.Errorf("error reading data file: %v", err) - return - } - filename := fi.Name() - ext := filepath.Ext(filename) - id := filename[0 : len(filename)-len(ext)] - err = index.Index(id, fileBytes) - if err != nil { - t.Errorf("error indexing data: %v", err) - return - } - } - - // read the searches - searchBytes, err := ioutil.ReadFile(dir + string(filepath.Separator) + "searches.json") - if err != nil { - t.Errorf("error reading searches: %v", err) - return - } - var searches SearchTests - err = json.Unmarshal(searchBytes, &searches) - if err != nil { - t.Errorf("error unmarshalling searches: %v", err) - return - } - - // run the searches - for testNum, search := range searches { - res, err := index.Search(search.Search) - if err != nil { - t.Errorf("error running search: %v", err) - } - if res.Total != search.Result.Total { - t.Errorf("test %d - expected total: %d got %d", testNum, search.Result.Total, res.Total) - continue - } - if len(res.Hits) != len(search.Result.Hits) { - t.Errorf("test %d - expected hits len: %d got %d", testNum, len(search.Result.Hits), len(res.Hits)) - continue - } - for hi, hit := range search.Result.Hits { - if hit.ID != res.Hits[hi].ID { - t.Errorf("test %d - expected hit %d to have ID %s got %s", testNum, hi, hit.ID, res.Hits[hi].ID) - } - if hit.Fields != nil { - if !reflect.DeepEqual(hit.Fields, res.Hits[hi].Fields) { - t.Errorf("test %d - expected hit %d to have fields %#v got %#v", testNum, hi, hit.Fields, res.Hits[hi].Fields) - } - } - if hit.Fragments != nil { - if !reflect.DeepEqual(hit.Fragments, res.Hits[hi].Fragments) { - t.Errorf("test %d - expected hit %d to have fragments %#v got %#v", testNum, hi, hit.Fragments, res.Hits[hi].Fragments) - } - } - } - if search.Result.Facets != nil { - if !reflect.DeepEqual(search.Result.Facets, res.Facets) { - t.Errorf("test %d - expected facets: %#v got %#v", testNum, search.Result.Facets, res.Facets) - } - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/a.json b/vendor/github.com/blevesearch/bleve/test/tests/basic/data/a.json deleted file mode 100644 index 992bbd99..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/a.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "id": "a", - "name": "marty", - "age": 19, - "title": "mista", - "tags": ["gopher", "belieber"] -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/b.json b/vendor/github.com/blevesearch/bleve/test/tests/basic/data/b.json deleted file mode 100644 index 58118a55..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/b.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "id": "b", - "name": "steve has a long name", - "age": 27, - "birthday": "2001-09-09T01:46:40Z", - "title": "missess" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/c.json b/vendor/github.com/blevesearch/bleve/test/tests/basic/data/c.json deleted file mode 100644 index 2b5d3088..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/c.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "id": "c", - "name": "bob walks home", - "age": 64, - "birthday": "2014-05-13T16:53:20Z", - "title": "masta" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/d.json b/vendor/github.com/blevesearch/bleve/test/tests/basic/data/d.json deleted file mode 100644 index dbc2f51b..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/basic/data/d.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "id": "d", - "name": "bobbleheaded wings top the phone", - "age": 72, - "birthday": "2014-05-13T16:53:20Z", - "title": "mizz" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/basic/mapping.json b/vendor/github.com/blevesearch/bleve/test/tests/basic/mapping.json deleted file mode 100644 index a7a049be..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/basic/mapping.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "types": { - "person": { - "properties": { - "name": { - "fields": [ - { - "include_term_vectors": true, - "include_in_all": true, - "index": true, - "store": true, - "analyzer": "en", - "type": "text" - } - ], - "dynamic": true, - "enabled": true - }, - "id": { - "dynamic": false, - "enabled": false - } - } - } - }, - "default_type": "person" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/basic/searches.json b/vendor/github.com/blevesearch/bleve/test/tests/basic/searches.json deleted file mode 100644 index 6f61fc1e..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/basic/searches.json +++ /dev/null @@ -1,406 +0,0 @@ -[ - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "name", - "term": "marti" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "name", - "term": "noone" - } - }, - "result": { - "total_hits": 0, - "hits": [] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "match_phrase": "long name" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "b" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "name", - "term": "walking" - } - }, - "result": { - "total_hits": 0, - "hits": [] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "fuzziness": 0, - "prefix_length": 0, - "field": "name", - "match": "walking" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "c" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "name", - "prefix": "bobble" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "d" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "query": "+name:phone" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "d" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "age", - "max": 30 - } - }, - "result": { - "total_hits": 2, - "hits": [ - { - "id": "b" - }, - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "age", - "max": 30, - "min": 20 - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "b" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "conjuncts": [ - { - "boost": 1, - "field": "age", - "min": 20 - }, - { - "boost": 1, - "field": "age", - "max": 30 - } - ] - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "b" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "birthday", - "start": "2010-01-01" - } - }, - "result": { - "total_hits": 2, - "hits": [ - { - "id": "d" - }, - { - "id": "c" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "birthday", - "end": "2010-01-01" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "b" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "tags", - "term": "gopher" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "tags", - "term": "belieber" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "tags", - "term": "notintagsarray" - } - }, - "result": { - "total_hits": 0, - "hits": [] - } - }, - { - "comment": "with size 0, total should be 1, but hits empty", - "search": { - "from": 0, - "size": 0, - "query": { - "field": "name", - "term": "marti" - } - }, - "result": { - "total_hits": 1, - "hits": [] - } - }, - { - "comment": "a search for doc a that includes tags field, verifies both values come back", - "search": { - "from": 0, - "size": 10, - "fields": ["tags"], - "query": { - "field": "name", - "term": "marti" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a", - "fields": { - "tags": ["gopher", "belieber"] - } - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "name", - "term": "msrti", - "fuzziness": 1 - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "comment": "highlight results", - "search": { - "from": 0, - "size": 10, - "query": { - "field": "name", - "match": "long" - }, - "highlight": { - "fields": ["name"] - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "b", - "fragments": { - "name": ["steve has a long name"] - } - } - ] - } - }, - { - "comment": "highlight results without specifying fields", - "search": { - "from": 0, - "size": 10, - "query": { - "field": "name", - "match": "long" - }, - "highlight": {} - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "b", - "fragments": { - "name": ["steve has a long name"] - } - } - ] - } - }, - { - "comment": "request fields", - "search": { - "from": 0, - "size": 10, - "fields": ["age","birthday"], - "query": { - "field": "name", - "match": "long" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "b", - "fields": { - "age": 27, - "birthday": "2001-09-09T01:46:40Z" - } - } - ] - } - } -] \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/a.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/a.json deleted file mode 100644 index 777b21f9..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/a.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "book", - "rating": 2, - "updated": "2014-11-25" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/b.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/b.json deleted file mode 100644 index c3c2c696..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/b.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "book", - "rating": 7, - "updated": "2013-07-25" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/c.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/c.json deleted file mode 100644 index da3542cc..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/c.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "book", - "rating": 1, - "updated": "2014-03-03" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/d.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/d.json deleted file mode 100644 index 20aaef1f..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/d.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "book", - "rating": 9, - "updated": "2014-09-16" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/e.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/e.json deleted file mode 100644 index 8dbd28a8..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/e.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "book", - "rating": 5, - "updated": "2014-11-15" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/f.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/f.json deleted file mode 100644 index 74e8cd21..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/f.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "movie", - "rating": 3, - "updated": "2017-06-05" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/g.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/g.json deleted file mode 100644 index ea5f29fc..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/g.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "movie", - "rating": 9, - "updated": "2011-10-03" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/h.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/h.json deleted file mode 100644 index f91c0d78..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/h.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "movie", - "rating": 9, - "updated": "2019-08-26" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/i.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/i.json deleted file mode 100644 index e46e5b70..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/i.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "movie", - "rating": 1, - "updated": "2014-12-14" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/j.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/data/j.json deleted file mode 100644 index 263f07b5..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/data/j.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "category": "inventory", - "type": "game", - "rating": 9, - "updated": "2013-10-20" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/mapping.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/mapping.json deleted file mode 100644 index 9e26dfee..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/mapping.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/facet/searches.json b/vendor/github.com/blevesearch/bleve/test/tests/facet/searches.json deleted file mode 100644 index 6752282a..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/facet/searches.json +++ /dev/null @@ -1,144 +0,0 @@ -[ - { - "search": { - "from": 0, - "size": 0, - "query": { - "field": "category", - "term": "inventory" - }, - "facets": { - "types": { - "size": 3, - "field": "type" - } - } - }, - "result": { - "total_hits": 10, - "hits": [], - "facets": { - "types": { - "field": "type", - "total": 10, - "missing": 0, - "other": 0, - "terms": [ - { - "term": "book", - "count": 5 - }, - { - "term": "movie", - "count": 4 - }, - { - "term": "game", - "count": 1 - } - ] - } - } - } - }, - { - "search": { - "from": 0, - "size": 0, - "query": { - "field": "category", - "term": "inventory" - }, - "facets": { - "types": { - "size": 3, - "field": "rating", - "numeric_ranges": [ - { - "name": "low", - "max": 5 - }, - { - "name": "high", - "min": 5 - } - ] - } - } - }, - "result": { - "total_hits": 10, - "hits": [], - "facets": { - "types": { - "field": "rating", - "total": 10, - "missing": 0, - "other": 0, - "numeric_ranges": [ - { - "name": "high", - "count": 6, - "min": 5 - }, - { - "name": "low", - "count": 4, - "max": 5 - } - ] - } - } - } - }, - { - "search": { - "from": 0, - "size": 0, - "query": { - "field": "category", - "term": "inventory" - }, - "facets": { - "types": { - "size": 3, - "field": "updated", - "date_ranges": [ - { - "name": "old", - "end": "2012-01-01" - }, - { - "name": "new", - "start": "2012-01-01" - } - ] - } - } - }, - "result": { - "total_hits": 10, - "hits": [], - "facets": { - "types": { - "field": "updated", - "total": 10, - "missing": 0, - "other": 0, - "date_ranges": [ - { - "name": "new", - "count": 9, - "start": "2012-01-01T00:00:00Z" - }, - { - "name": "old", - "count": 1, - "end": "2012-01-01T00:00:00Z" - } - ] - } - } - } - } -] \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json b/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json deleted file mode 100644 index 62918db4..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3311@FOSDEM15@fosdem.org.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "description": "From Prolog to Erlang to Haskell to Lisp to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful", - "category": "Word" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json b/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json deleted file mode 100644 index 874f60ec..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3492@FOSDEM15@fosdem.org.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "description": "different cats", - "category": "Perl" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json b/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json deleted file mode 100644 index 1acdf344..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3496@FOSDEM15@fosdem.org.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "description": "many cats", - "category": "Perl" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json b/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json deleted file mode 100644 index fb670aa2..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3505@FOSDEM15@fosdem.org.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "description": "From Prolog to Erlang to Haskell to Lisp to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful", - "category": "Perl" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json b/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json deleted file mode 100644 index 38be8e35..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/data/3507@FOSDEM15@fosdem.org.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "description": "From Prolog to Erlang to Haskell to Gel to TLC and then back to Prolog I have journeyed, and I'd like to share some of the beautiful", - "category": "Perl" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/mapping.json b/vendor/github.com/blevesearch/bleve/test/tests/fosdem/mapping.json deleted file mode 100644 index 470b9940..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/mapping.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "default_mapping": { - "enabled": true, - "dynamic": true, - "properties": { - "category": { - "enabled": true, - "dynamic": true, - "fields": [ - { - "type": "text", - "analyzer": "keyword", - "store": true, - "index": true, - "include_term_vectors": true, - "include_in_all": true - } - ], - "default_analyzer": "" - }, - "description": { - "enabled": true, - "dynamic": true, - "fields": [ - { - "type": "text", - "analyzer": "en", - "store": true, - "index": true, - "include_term_vectors": true, - "include_in_all": true - } - ], - "default_analyzer": "" - }, - "summary": { - "enabled": true, - "dynamic": true, - "fields": [ - { - "type": "text", - "analyzer": "en", - "store": true, - "index": true, - "include_term_vectors": true, - "include_in_all": true - } - ], - "default_analyzer": "" - }, - "url": { - "enabled": true, - "dynamic": true, - "fields": [ - { - "type": "text", - "analyzer": "keyword", - "store": true, - "index": true, - "include_term_vectors": true, - "include_in_all": true - } - ], - "default_analyzer": "" - } - }, - "default_analyzer": "" - }, - "type_field": "_type", - "default_type": "_default", - "default_analyzer": "en", - "default_datetime_parser": "dateTimeOptional", - "default_field": "_all", - "byte_array_converter": "json", - "analysis": {} -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/searches.json b/vendor/github.com/blevesearch/bleve/test/tests/fosdem/searches.json deleted file mode 100644 index d7ed9efd..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/fosdem/searches.json +++ /dev/null @@ -1,105 +0,0 @@ -[ - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "category", - "match_phrase": "Perl" - } - }, - "result": { - "total_hits": 4, - "hits": [ - { - "id": "3507@FOSDEM15@fosdem.org" - }, - { - "id": "3505@FOSDEM15@fosdem.org" - }, - { - "id": "3496@FOSDEM15@fosdem.org" - } - , - { - "id": "3492@FOSDEM15@fosdem.org" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "match": "lisp" - } - }, - "result": { - "total_hits": 2, - "hits": [ - { - "id": "3505@FOSDEM15@fosdem.org" - }, - { - "id": "3311@FOSDEM15@fosdem.org" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": {"boost":1,"query":"+lisp +category:Perl"} - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "3505@FOSDEM15@fosdem.org" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": {"boost":1,"query":"+lisp +category:\"Perl\""} - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "3505@FOSDEM15@fosdem.org" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "must": { - "conjuncts":[ - {"boost":1,"query":"+cats"}, - {"field":"category","match_phrase":"Perl"} - ] - } - } - }, - "result": { - "total_hits": 2, - "hits": [ - { - "id": "3496@FOSDEM15@fosdem.org" - }, - { - "id": "3492@FOSDEM15@fosdem.org" - } - ] - } - } -] \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/phrase/data/a.json b/vendor/github.com/blevesearch/bleve/test/tests/phrase/data/a.json deleted file mode 100644 index 5fe2dd87..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/phrase/data/a.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "body": "Twenty Thousand Leagues Under The Sea" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/phrase/mapping.json b/vendor/github.com/blevesearch/bleve/test/tests/phrase/mapping.json deleted file mode 100644 index 6629a21c..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/phrase/mapping.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "types": { - "book": { - "properties": { - "body": { - "fields": [ - { - "include_term_vectors": true, - "include_in_all": true, - "index": true, - "store": true, - "analyzer": "en", - "type": "text" - } - ], - "dynamic": true, - "enabled": true - } - } - } - }, - "default_type": "book" -} \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/test/tests/phrase/searches.json b/vendor/github.com/blevesearch/bleve/test/tests/phrase/searches.json deleted file mode 100644 index 76c0d474..00000000 --- a/vendor/github.com/blevesearch/bleve/test/tests/phrase/searches.json +++ /dev/null @@ -1,326 +0,0 @@ -[ - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Twenty" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Twenty Thousand" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Twenty Thousand Leagues" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Twenty Thousand Leagues Under" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Twenty Thousand Leagues Under the" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Twenty Thousand Leagues Under the Sea" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Thousand" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Thousand Leagues" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Thousand Leagues Under" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Thousand Leagues Under the" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Thousand Leagues Under the Sea" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Leagues" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Leagues Under" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Leagues Under the" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Leagues Under the Sea" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Under the Sea" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "the Sea" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - }, - { - "search": { - "from": 0, - "size": 10, - "query": { - "field": "body", - "match_phrase": "Sea" - } - }, - "result": { - "total_hits": 1, - "hits": [ - { - "id": "a" - } - ] - } - } -] \ No newline at end of file diff --git a/vendor/github.com/blevesearch/bleve/utils/bleve_bulkindex/main.go b/vendor/github.com/blevesearch/bleve/utils/bleve_bulkindex/main.go deleted file mode 100644 index 5131d841..00000000 --- a/vendor/github.com/blevesearch/bleve/utils/bleve_bulkindex/main.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package main - -import ( - "bufio" - "flag" - "log" - "math/rand" - "os" - - "github.com/blevesearch/bleve" -) - -var indexPath = flag.String("index", "", "index path") -var batchSize = flag.Int("size", 1000, "size of a single batch to index") - -func main() { - - flag.Parse() - - if *indexPath == "" { - log.Fatal("must specify index path") - } - - // open the index - index, err := bleve.Open(*indexPath) - if err != nil { - log.Fatal(err) - } - defer func() { - cerr := index.Close() - if cerr != nil { - log.Fatalf("error closing index: %v", err) - } - }() - - if flag.NArg() < 1 { - log.Fatal("must specify at least one path to index") - } - - i := 0 - batch := index.NewBatch() - - for _, file := range flag.Args() { - - file, err := os.Open(file) - defer func() { - cerr := file.Close() - if cerr != nil { - log.Fatalf("error closing file: %v", cerr) - } - }() - if err != nil { - log.Fatal(err) - } - - log.Printf("Indexing: %s\n", file.Name()) - r := bufio.NewReader(file) - - for { - if i%*batchSize == 0 { - log.Printf("Indexing batch (%d docs)...\n", i) - err := index.Batch(batch) - if err != nil { - log.Fatal(err) - } - batch = index.NewBatch() - } - - b, _ := r.ReadBytes('\n') - if len(b) == 0 { - break - } - docID := randomString(5) - err := batch.Index(docID, b) - if err != nil { - log.Fatal(err) - } - i++ - } - err = index.Batch(batch) - if err != nil { - log.Fatal(err) - } - } -} - -var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randomString(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return string(b) -} diff --git a/vendor/github.com/blevesearch/bleve/utils/bleve_create/main.go b/vendor/github.com/blevesearch/bleve/utils/bleve_create/main.go deleted file mode 100644 index 9f940b16..00000000 --- a/vendor/github.com/blevesearch/bleve/utils/bleve_create/main.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package main - -import ( - "encoding/json" - "flag" - "io/ioutil" - "log" - - "github.com/blevesearch/bleve" -) - -var indexPath = flag.String("index", "", "index path") -var mappingFile = flag.String("mapping", "", "mapping file") - -func main() { - - flag.Parse() - - if *indexPath == "" { - log.Fatal("must specify index path") - } - - // create a new default mapping - mapping := bleve.NewIndexMapping() - if *mappingFile != "" { - mappingBytes, err := ioutil.ReadFile(*mappingFile) - if err != nil { - log.Fatal(err) - } - err = json.Unmarshal(mappingBytes, &mapping) - if err != nil { - log.Fatal(err) - } - } - - // create the index - index, err := bleve.New(*indexPath, mapping) - if err != nil { - log.Fatal(err) - } - defer func() { - cerr := index.Close() - if cerr != nil { - log.Fatalf("error closing index: %v", err) - } - }() - - log.Printf("Created bleve index at: %s", *indexPath) -} diff --git a/vendor/github.com/blevesearch/bleve/utils/bleve_dump/main.go b/vendor/github.com/blevesearch/bleve/utils/bleve_dump/main.go deleted file mode 100644 index 33174d30..00000000 --- a/vendor/github.com/blevesearch/bleve/utils/bleve_dump/main.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package main - -import ( - "encoding/json" - "flag" - "fmt" - "log" - - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/index/upside_down" -) - -var indexPath = flag.String("index", "", "index path") - -var fieldsOnly = flag.Bool("fields", false, "fields only") -var docID = flag.String("docID", "", "docID to dump") -var mappingOnly = flag.Bool("mapping", false, "print mapping") - -func main() { - flag.Parse() - if *indexPath == "" { - log.Fatal("specify index to dump") - } - - index, err := bleve.Open(*indexPath) - if err != nil { - log.Fatal(err) - } - defer func() { - cerr := index.Close() - if cerr != nil { - log.Fatalf("error closing index: %v", err) - } - }() - - if *mappingOnly { - mapping := index.Mapping() - jsonBytes, err := json.MarshalIndent(mapping, "", " ") - if err != nil { - log.Fatal(err) - } - fmt.Printf("%s\n", jsonBytes) - return - } - - var dumpChan chan interface{} - if *docID != "" { - dumpChan = index.DumpDoc(*docID) - } else if *fieldsOnly { - dumpChan = index.DumpFields() - } else { - dumpChan = index.DumpAll() - } - - for rowOrErr := range dumpChan { - switch rowOrErr := rowOrErr.(type) { - case error: - log.Printf("error dumping: %v", rowOrErr) - case upside_down.UpsideDownCouchRow: - fmt.Printf("%v\n", rowOrErr) - fmt.Printf("Key: % -100x\nValue: % -100x\n\n", rowOrErr.Key(), rowOrErr.Value()) - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/utils/bleve_index/main.go b/vendor/github.com/blevesearch/bleve/utils/bleve_index/main.go deleted file mode 100644 index 54008786..00000000 --- a/vendor/github.com/blevesearch/bleve/utils/bleve_index/main.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package main - -import ( - "flag" - "io/ioutil" - "log" - "os" - "path/filepath" - - "github.com/blevesearch/bleve" -) - -var indexPath = flag.String("index", "", "index path") -var keepExt = flag.Bool("keepExt", false, "keep extension in doc id") -var keepDir = flag.Bool("keepDir", false, "keep dir in doc id") - -func main() { - - flag.Parse() - - if *indexPath == "" { - log.Fatal("must specify index path") - } - - // open the index - index, err := bleve.Open(*indexPath) - if err != nil { - log.Fatal(err) - } - defer func() { - cerr := index.Close() - if cerr != nil { - log.Fatalf("error closing index: %v", err) - } - }() - - if flag.NArg() < 1 { - log.Fatal("must specify at least one path to index") - } - - for file := range handleArgs(flag.Args()) { - // index the files - docID := file.filename - if !*keepDir { - _, docID = filepath.Split(docID) - } - if !*keepExt { - ext := filepath.Ext(docID) - docID = docID[0 : len(docID)-len(ext)] - } - log.Printf("Indexing: %s", docID) - err = index.Index(docID, file.contents) - if err != nil { - log.Fatal(err) - } - } -} - -type file struct { - filename string - contents []byte -} - -func handleArgs(args []string) chan file { - rv := make(chan file) - - go func() { - for _, arg := range args { - arg = filepath.Clean(arg) - handleArgRecursive(arg, rv) - } - close(rv) - }() - - return rv -} - -func handleArgRecursive(arg string, results chan file) { - stat, err := os.Stat(arg) - if err != nil { - log.Print(err) - return - } - if stat.IsDir() { - // open the directory - dirEntries, err := ioutil.ReadDir(arg) - if err != nil { - log.Fatal(err) - } - - // walk the directory entries - for _, dirEntry := range dirEntries { - handleArgRecursive(arg+string(os.PathSeparator)+dirEntry.Name(), results) - } - } else { - bytes, err := ioutil.ReadFile(arg) - if err != nil { - log.Fatal(err) - } - - results <- file{ - filename: arg, - contents: bytes, - } - } -} diff --git a/vendor/github.com/blevesearch/bleve/utils/bleve_query/main.go b/vendor/github.com/blevesearch/bleve/utils/bleve_query/main.go deleted file mode 100644 index f68175ef..00000000 --- a/vendor/github.com/blevesearch/bleve/utils/bleve_query/main.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package main - -import ( - "flag" - "fmt" - "log" - "os" - "runtime/pprof" - "strings" - - "github.com/blevesearch/bleve" -) - -var indexPath = flag.String("index", "", "index path") -var limit = flag.Int("limit", 10, "limit to first N results") -var skip = flag.Int("skip", 0, "skip the first N results") -var explain = flag.Bool("explain", false, "explain scores") -var includeHighlights = flag.Bool("highlight", true, "highlight matches") -var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") -var repeat = flag.Int("repeat", 1, "repeat query n times") - -func main() { - - flag.Parse() - - if *cpuprofile != "" { - f, err := os.Create(*cpuprofile) - if err != nil { - log.Fatal(err) - } - err = pprof.StartCPUProfile(f) - if err != nil { - log.Fatal(err) - } - defer pprof.StopCPUProfile() - } - - if *indexPath == "" { - log.Fatal("Specify index to query") - } - - if flag.NArg() < 1 { - log.Fatal("Specify search query") - } - - // open index - index, err := bleve.Open(*indexPath) - if err != nil { - log.Fatal(err) - } - defer func() { - cerr := index.Close() - if cerr != nil { - log.Fatalf("error closing index: %v", err) - } - }() - - for i := 0; i < *repeat; i++ { - // build a search with the provided parameters - queryString := strings.Join(flag.Args(), " ") - query := bleve.NewQueryStringQuery(queryString) - searchRequest := bleve.NewSearchRequestOptions(query, *limit, *skip, *explain) - - // enable highlights if requested - if *includeHighlights { - searchRequest.Highlight = bleve.NewHighlightWithStyle("ansi") - } - - // execute the search - searchResult, err := index.Search(searchRequest) - if err != nil { - log.Fatalf("search error: %v", err) - } - fmt.Println(searchResult) - } -} diff --git a/vendor/github.com/blevesearch/bleve/utils/bleve_registry/main.go b/vendor/github.com/blevesearch/bleve/utils/bleve_registry/main.go deleted file mode 100644 index 9857532f..00000000 --- a/vendor/github.com/blevesearch/bleve/utils/bleve_registry/main.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package main - -import ( - "fmt" - "sort" - - _ "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/registry" -) - -func main() { - fmt.Printf("Bleve Registry:\n") - printRegistry() -} - -func printRegistry() { - types, instances := registry.CharFilterTypesAndInstances() - printType("Char Filter", types, instances) - - types, instances = registry.TokenizerTypesAndInstances() - printType("Tokenizer", types, instances) - - types, instances = registry.TokenMapTypesAndInstances() - printType("Token Map", types, instances) - - types, instances = registry.TokenFilterTypesAndInstances() - printType("Token Filter", types, instances) - - types, instances = registry.AnalyzerTypesAndInstances() - printType("Analyzer", types, instances) - - types, instances = registry.DateTimeParserTypesAndInstances() - printType("Date Time Parser", types, instances) - - types, instances = registry.KVStoreTypesAndInstances() - printType("KV Store", types, instances) - - types, instances = registry.ByteArrayConverterTypesAndInstances() - printType("ByteArrayConverter", types, instances) - - types, instances = registry.FragmentFormatterTypesAndInstances() - printType("Fragment Formatter", types, instances) - - types, instances = registry.FragmenterTypesAndInstances() - printType("Fragmenter", types, instances) - - types, instances = registry.HighlighterTypesAndInstances() - printType("Highlighter", types, instances) -} - -func sortStrings(in []string) []string { - sortedStrings := make(sort.StringSlice, 0, len(in)) - for _, str := range in { - sortedStrings = append(sortedStrings, str) - } - sortedStrings.Sort() - return sortedStrings -} - -func printType(label string, types, instances []string) { - sortedTypes := sortStrings(types) - sortedInstances := sortStrings(instances) - fmt.Printf(label + " Types:\n") - for _, name := range sortedTypes { - fmt.Printf("\t%s\n", name) - } - fmt.Println() - fmt.Printf(label + " Instances:\n") - for _, name := range sortedInstances { - fmt.Printf("\t%s\n", name) - } - fmt.Println() -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_contains_vowel_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_contains_vowel_test.go deleted file mode 100644 index 3532cb7d..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_contains_vowel_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestContainsVowel(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected bool - }, 15) - - tests[i].S = []rune("apple") - tests[i].Expected = true - i++ - - tests[i].S = []rune("f") - tests[i].Expected = false - i++ - - tests[i].S = []rune("a") - tests[i].Expected = true - i++ - - tests[i].S = []rune("e") - tests[i].Expected = true - i++ - - tests[i].S = []rune("i") - tests[i].Expected = true - i++ - - tests[i].S = []rune("o") - tests[i].Expected = true - i++ - - tests[i].S = []rune("u") - tests[i].Expected = true - i++ - - tests[i].S = []rune("y") - tests[i].Expected = false - i++ - - tests[i].S = []rune("cy") - tests[i].Expected = true - i++ - - for _, datum := range tests { - if actual := containsVowel(datum.S); actual != datum.Expected { - t.Errorf("Did NOT get what was expected for calling containsVowel() on [%s]. Expect [%t] but got [%t]", string(datum.S), datum.Expected, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fixes_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fixes_test.go deleted file mode 100644 index a99eda9e..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fixes_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -// Test for issue listed here: -// https://github.com/reiver/go-porterstemmer/issues/1 -// -// StemString("ion") was causing runtime exception -func TestStemStringIon(t *testing.T) { - - expected := "ion" - - s := "ion" - actual := StemString(s) - if expected != actual { - t.Errorf("Input: [%s] -> Actual: [%s]. Expected: [%s]", s, actual, expected) - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fuzz_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fuzz_test.go deleted file mode 100644 index af398390..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_fuzz_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package porterstemmer - -import ( - "bytes" - "testing" -) - -const maxFuzzLen = 6 - -// Test inputs of English characters less than maxFuzzLen -// Added to help diagnose https://github.com/reiver/go-porterstemmer/issues/4 -func TestStemFuzz(t *testing.T) { - - input := []byte{'a'} - for len(input) < maxFuzzLen { - // test input - - panicked := false - func() { - defer func() { panicked = recover() != nil }() - StemString(string(input)) - }() - if panicked { - t.Errorf("StemString panicked for input '%s'", input) - } - - // if all z's extend - if allZs(input) { - input = bytes.Repeat([]byte{'a'}, len(input)+1) - } else { - // increment - input = incrementBytes(input) - } - } -} - -func incrementBytes(in []byte) []byte { - rv := make([]byte, len(in)) - copy(rv, in) - for i := len(rv) - 1; i >= 0; i-- { - if rv[i]+1 == '{' { - rv[i] = 'a' - continue - } - rv[i] = rv[i] + 1 - break - - } - return rv -} - -func allZs(in []byte) bool { - for _, b := range in { - if b != 'z' { - return false - } - } - return true -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_repeat_double_consonant_suffix_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_repeat_double_consonant_suffix_test.go deleted file mode 100644 index 420f84a1..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_repeat_double_consonant_suffix_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestHasDoubleConsonantSuffix(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected bool - }, 12) - - tests[i].S = []rune("apple") - tests[i].Expected = false - i++ - - tests[i].S = []rune("hiss") - tests[i].Expected = true - i++ - - tests[i].S = []rune("fizz") - tests[i].Expected = true - i++ - - tests[i].S = []rune("fill") - tests[i].Expected = true - i++ - - tests[i].S = []rune("ahaa") - tests[i].Expected = false - i++ - - for _, datum := range tests { - - if actual := hasRepeatDoubleConsonantSuffix(datum.S); actual != datum.Expected { - t.Errorf("Did NOT get what was expected for calling hasDoubleConsonantSuffix() on [%s]. Expect [%t] but got [%t]", string(datum.S), datum.Expected, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_suffix_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_suffix_test.go deleted file mode 100644 index 47ae81b5..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_has_suffix_test.go +++ /dev/null @@ -1,432 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestHasSuffix(t *testing.T) { - - tests := make([]struct { - S []rune - Suffix []rune - Expected bool - }, 82) - - i := 0 - - tests[i].S = []rune("ran") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - tests[i].S = []rune("runner") - tests[i].Suffix = []rune("er") - tests[i].Expected = true - i++ - - tests[i].S = []rune("runnar") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - tests[i].S = []rune("runned") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - tests[i].S = []rune("runnre") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - // FIXME marty changed Expected - // to false here because it seems - // the contract does not support - // suffix of same length as input - // as this test implied - tests[i].S = []rune("er") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - tests[i].S = []rune("re") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - tests[i].S = []rune("ran") - tests[i].Suffix = []rune("ER") - tests[i].Expected = false - i++ - - tests[i].S = []rune("runner") - tests[i].Suffix = []rune("ER") - tests[i].Expected = false - i++ - - tests[i].S = []rune("runnar") - tests[i].Suffix = []rune("ER") - tests[i].Expected = false - i++ - - tests[i].S = []rune("runned") - tests[i].Suffix = []rune("ER") - tests[i].Expected = false - i++ - - tests[i].S = []rune("runnre") - tests[i].Suffix = []rune("ER") - tests[i].Expected = false - i++ - - tests[i].S = []rune("er") - tests[i].Suffix = []rune("ER") - tests[i].Expected = false - i++ - - tests[i].S = []rune("re") - tests[i].Suffix = []rune("ER") - tests[i].Expected = false - i++ - - tests[i].S = []rune("") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - tests[i].S = []rune("e") - tests[i].Suffix = []rune("er") - tests[i].Expected = false - i++ - - tests[i].S = []rune("caresses") - tests[i].Suffix = []rune("sses") - tests[i].Expected = true - i++ - - tests[i].S = []rune("ponies") - tests[i].Suffix = []rune("ies") - tests[i].Expected = true - i++ - - tests[i].S = []rune("caress") - tests[i].Suffix = []rune("ss") - tests[i].Expected = true - i++ - - tests[i].S = []rune("cats") - tests[i].Suffix = []rune("s") - tests[i].Expected = true - i++ - - tests[i].S = []rune("feed") - tests[i].Suffix = []rune("eed") - tests[i].Expected = true - i++ - - tests[i].S = []rune("agreed") - tests[i].Suffix = []rune("eed") - tests[i].Expected = true - i++ - - tests[i].S = []rune("plastered") - tests[i].Suffix = []rune("ed") - tests[i].Expected = true - i++ - - tests[i].S = []rune("bled") - tests[i].Suffix = []rune("ed") - tests[i].Expected = true - i++ - - tests[i].S = []rune("motoring") - tests[i].Suffix = []rune("ing") - tests[i].Expected = true - i++ - - tests[i].S = []rune("sing") - tests[i].Suffix = []rune("ing") - tests[i].Expected = true - i++ - - tests[i].S = []rune("conflat") - tests[i].Suffix = []rune("at") - tests[i].Expected = true - i++ - - tests[i].S = []rune("troubl") - tests[i].Suffix = []rune("bl") - tests[i].Expected = true - i++ - - tests[i].S = []rune("siz") - tests[i].Suffix = []rune("iz") - tests[i].Expected = true - i++ - - tests[i].S = []rune("happy") - tests[i].Suffix = []rune("y") - tests[i].Expected = true - i++ - - tests[i].S = []rune("sky") - tests[i].Suffix = []rune("y") - tests[i].Expected = true - i++ - - tests[i].S = []rune("relational") - tests[i].Suffix = []rune("ational") - tests[i].Expected = true - i++ - - tests[i].S = []rune("conditional") - tests[i].Suffix = []rune("tional") - tests[i].Expected = true - i++ - - tests[i].S = []rune("rational") - tests[i].Suffix = []rune("tional") - tests[i].Expected = true - i++ - - tests[i].S = []rune("valenci") - tests[i].Suffix = []rune("enci") - tests[i].Expected = true - i++ - - tests[i].S = []rune("hesitanci") - tests[i].Suffix = []rune("anci") - tests[i].Expected = true - i++ - - tests[i].S = []rune("digitizer") - tests[i].Suffix = []rune("izer") - tests[i].Expected = true - i++ - - tests[i].S = []rune("conformabli") - tests[i].Suffix = []rune("abli") - tests[i].Expected = true - i++ - - tests[i].S = []rune("radicalli") - tests[i].Suffix = []rune("alli") - tests[i].Expected = true - i++ - - tests[i].S = []rune("differentli") - tests[i].Suffix = []rune("entli") - tests[i].Expected = true - i++ - - tests[i].S = []rune("vileli") - tests[i].Suffix = []rune("eli") - tests[i].Expected = true - i++ - - tests[i].S = []rune("analogousli") - tests[i].Suffix = []rune("ousli") - tests[i].Expected = true - i++ - - tests[i].S = []rune("vietnamization") - tests[i].Suffix = []rune("ization") - tests[i].Expected = true - i++ - - tests[i].S = []rune("predication") - tests[i].Suffix = []rune("ation") - tests[i].Expected = true - i++ - - tests[i].S = []rune("operator") - tests[i].Suffix = []rune("ator") - tests[i].Expected = true - i++ - - tests[i].S = []rune("feudalism") - tests[i].Suffix = []rune("alism") - tests[i].Expected = true - i++ - - tests[i].S = []rune("decisiveness") - tests[i].Suffix = []rune("iveness") - tests[i].Expected = true - i++ - - tests[i].S = []rune("hopefulness") - tests[i].Suffix = []rune("fulness") - tests[i].Expected = true - i++ - - tests[i].S = []rune("callousness") - tests[i].Suffix = []rune("ousness") - tests[i].Expected = true - i++ - - tests[i].S = []rune("formaliti") - tests[i].Suffix = []rune("aliti") - tests[i].Expected = true - i++ - - tests[i].S = []rune("sensitiviti") - tests[i].Suffix = []rune("iviti") - tests[i].Expected = true - i++ - - tests[i].S = []rune("sensibiliti") - tests[i].Suffix = []rune("biliti") - tests[i].Expected = true - i++ - - tests[i].S = []rune("triplicate") - tests[i].Suffix = []rune("icate") - tests[i].Expected = true - i++ - - tests[i].S = []rune("formative") - tests[i].Suffix = []rune("ative") - tests[i].Expected = true - i++ - - tests[i].S = []rune("formalize") - tests[i].Suffix = []rune("alize") - tests[i].Expected = true - i++ - - tests[i].S = []rune("electriciti") - tests[i].Suffix = []rune("iciti") - tests[i].Expected = true - i++ - - tests[i].S = []rune("electrical") - tests[i].Suffix = []rune("ical") - tests[i].Expected = true - i++ - - tests[i].S = []rune("hopeful") - tests[i].Suffix = []rune("ful") - tests[i].Expected = true - i++ - - tests[i].S = []rune("goodness") - tests[i].Suffix = []rune("ness") - tests[i].Expected = true - i++ - - tests[i].S = []rune("revival") - tests[i].Suffix = []rune("al") - tests[i].Expected = true - i++ - - tests[i].S = []rune("allowance") - tests[i].Suffix = []rune("ance") - tests[i].Expected = true - i++ - - tests[i].S = []rune("inference") - tests[i].Suffix = []rune("ence") - tests[i].Expected = true - i++ - - tests[i].S = []rune("airliner") - tests[i].Suffix = []rune("er") - tests[i].Expected = true - i++ - - tests[i].S = []rune("gyroscopic") - tests[i].Suffix = []rune("ic") - tests[i].Expected = true - i++ - - tests[i].S = []rune("adjustable") - tests[i].Suffix = []rune("able") - tests[i].Expected = true - i++ - - tests[i].S = []rune("defensible") - tests[i].Suffix = []rune("ible") - tests[i].Expected = true - i++ - - tests[i].S = []rune("irritant") - tests[i].Suffix = []rune("ant") - tests[i].Expected = true - i++ - - tests[i].S = []rune("replacement") - tests[i].Suffix = []rune("ement") - tests[i].Expected = true - i++ - - tests[i].S = []rune("adjustment") - tests[i].Suffix = []rune("ment") - tests[i].Expected = true - i++ - - tests[i].S = []rune("dependent") - tests[i].Suffix = []rune("ent") - tests[i].Expected = true - i++ - - tests[i].S = []rune("adoption") - tests[i].Suffix = []rune("ion") - tests[i].Expected = true - i++ - - tests[i].S = []rune("homologou") - tests[i].Suffix = []rune("ou") - tests[i].Expected = true - i++ - - tests[i].S = []rune("communism") - tests[i].Suffix = []rune("ism") - tests[i].Expected = true - i++ - - tests[i].S = []rune("activate") - tests[i].Suffix = []rune("ate") - tests[i].Expected = true - i++ - - tests[i].S = []rune("angulariti") - tests[i].Suffix = []rune("iti") - tests[i].Expected = true - i++ - - tests[i].S = []rune("homologous") - tests[i].Suffix = []rune("ous") - tests[i].Expected = true - i++ - - tests[i].S = []rune("effective") - tests[i].Suffix = []rune("ive") - tests[i].Expected = true - i++ - - tests[i].S = []rune("bowdlerize") - tests[i].Suffix = []rune("ize") - tests[i].Expected = true - i++ - - tests[i].S = []rune("probate") - tests[i].Suffix = []rune("e") - tests[i].Expected = true - i++ - - tests[i].S = []rune("rate") - tests[i].Suffix = []rune("e") - tests[i].Expected = true - i++ - - tests[i].S = []rune("cease") - tests[i].Suffix = []rune("e") - tests[i].Expected = true - i++ - - for _, datum := range tests { - if actual := hasSuffix(datum.S, datum.Suffix); actual != datum.Expected { - t.Errorf("Did NOT get what was expected for calling hasSuffix() on [%s] with suffix [%s]. Expect [%t] but got [%t]", string(datum.S), string(datum.Suffix), datum.Expected, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_is_consontant_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_is_consontant_test.go deleted file mode 100644 index dc665654..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_is_consontant_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestIsConsontant(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []bool - }, 12) - - tests[i].S = []rune("apple") - tests[i].Expected = []bool{false, true, true, true, false} - i++ - - tests[i].S = []rune("cyan") - tests[i].Expected = []bool{true, false, false, true} - i++ - - tests[i].S = []rune("connects") - tests[i].Expected = []bool{true, false, true, true, false, true, true, true} - i++ - - tests[i].S = []rune("yellow") - tests[i].Expected = []bool{true, false, true, true, false, true} - i++ - - tests[i].S = []rune("excellent") - tests[i].Expected = []bool{false, true, true, false, true, true, false, true, true} - i++ - - tests[i].S = []rune("yuk") - tests[i].Expected = []bool{true, false, true} - i++ - - tests[i].S = []rune("syzygy") - tests[i].Expected = []bool{true, false, true, false, true, false} - i++ - - tests[i].S = []rune("school") - tests[i].Expected = []bool{true, true, true, false, false, true} - i++ - - tests[i].S = []rune("pay") - tests[i].Expected = []bool{true, false, true} - i++ - - tests[i].S = []rune("golang") - tests[i].Expected = []bool{true, false, true, false, true, true} - i++ - - // NOTE: The Porter Stemmer technical should make a mistake on the second "y". - // Really, both the 1st and 2nd "y" are consontants. But - tests[i].S = []rune("sayyid") - tests[i].Expected = []bool{true, false, true, false, false, true} - i++ - - tests[i].S = []rune("ya") - tests[i].Expected = []bool{true, false} - i++ - - for _, datum := range tests { - for i = 0; i < len(datum.S); i++ { - - if actual := isConsonant(datum.S, i); actual != datum.Expected[i] { - t.Errorf("Did NOT get what was expected for calling isConsonant() on [%s] at [%d] (i.e., [%s]). Expect [%t] but got [%t]", string(datum.S), i, string(datum.S[i]), datum.Expected[i], actual) - } - } // for - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_measure_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_measure_test.go deleted file mode 100644 index d53024b9..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_measure_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestMeasure(t *testing.T) { - - tests := make([]struct { - S []rune - Expected uint - }, 27) - - tests[0].S = []rune("ya") - tests[0].Expected = 0 - - tests[1].S = []rune("cyan") - tests[1].Expected = 1 - - tests[2].S = []rune("connects") - tests[2].Expected = 2 - - tests[3].S = []rune("yellow") - tests[3].Expected = 2 - - tests[4].S = []rune("excellent") - tests[4].Expected = 3 - - tests[5].S = []rune("yuk") - tests[5].Expected = 1 - - tests[6].S = []rune("syzygy") - tests[6].Expected = 2 - - tests[7].S = []rune("school") - tests[7].Expected = 1 - - tests[8].S = []rune("pay") - tests[8].Expected = 1 - - tests[9].S = []rune("golang") - tests[9].Expected = 2 - - // NOTE: The Porter Stemmer technical should make a mistake on the second "y". - // Really, both the 1st and 2nd "y" are consontants. But - tests[10].S = []rune("sayyid") - tests[10].Expected = 2 - - tests[11].S = []rune("ya") - tests[11].Expected = 0 - - tests[12].S = []rune("") - tests[12].Expected = 0 - - tests[13].S = []rune("tr") - tests[13].Expected = 0 - - tests[14].S = []rune("ee") - tests[14].Expected = 0 - - tests[15].S = []rune("tree") - tests[15].Expected = 0 - - tests[16].S = []rune("t") - tests[16].Expected = 0 - - tests[18].S = []rune("by") - tests[18].Expected = 0 - - tests[19].S = []rune("trouble") - tests[19].Expected = 1 - - tests[20].S = []rune("oats") - tests[20].Expected = 1 - - tests[21].S = []rune("trees") - tests[21].Expected = 1 - - tests[22].S = []rune("ivy") - tests[22].Expected = 1 - - tests[23].S = []rune("troubles") - tests[23].Expected = 2 - - tests[24].S = []rune("private") - tests[24].Expected = 2 - - tests[25].S = []rune("oaten") - tests[25].Expected = 2 - - tests[26].S = []rune("orrery") - tests[26].Expected = 2 - - for _, datum := range tests { - if actual := measure(datum.S); actual != datum.Expected { - t.Errorf("Did NOT get what was expected for calling measure() on [%s]. Expect [%d] but got [%d]", string(datum.S), datum.Expected, actual) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_string_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_string_test.go deleted file mode 100644 index f93258d7..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_string_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package porterstemmer - -import ( - "bufio" - "io/ioutil" - "net/http" - "os" - "strings" - "testing" -) - -func TestStemString(t *testing.T) { - - testDataDirName := "testdata" - - _, err := os.Stat(testDataDirName) - if nil != err { - _ = os.Mkdir(testDataDirName, 0755) - } - _, err = os.Stat(testDataDirName) - if nil != err { - t.Errorf("The test data folder ([%s]) does not exists (and could not create it). Received error: [%v]", testDataDirName, err) - /////// RETURN - return - } - - vocFileName := testDataDirName + "/voc.txt" - _, err = os.Stat(vocFileName) - if nil != err { - - vocHref := "http://tartarus.org/martin/PorterStemmer/voc.txt" - - resp, err := http.Get(vocHref) - if nil != err { - t.Errorf("Could not download test file (from web) from URL: [%s]. Received error: [%v]", vocHref, err) - /////////// RETURN - return - } - - respBody, err := ioutil.ReadAll(resp.Body) - if nil != err { - t.Errorf("Error loading the contents of from URL: [%s]. Received error: [%v].", vocHref, err) - /////////// RETURN - return - } - - _ = ioutil.WriteFile(vocFileName, respBody, 0644) - - } - vocFd, err := os.Open(vocFileName) - if nil != err { - t.Errorf("Could NOT open testdata file: [%s]. Received error: [%v]", vocFileName, err) - /////// RETURN - return - } - defer vocFd.Close() - - voc := bufio.NewReaderSize(vocFd, 1024) - - outFileName := testDataDirName + "/output.txt" - _, err = os.Stat(outFileName) - if nil != err { - - outHref := "http://tartarus.org/martin/PorterStemmer/output.txt" - - resp, err := http.Get(outHref) - if nil != err { - t.Errorf("Could not download test file (from web) from URL: [%s]. Received error: [%v]", outHref, err) - /////////// RETURN - return - } - - respBody, err := ioutil.ReadAll(resp.Body) - if nil != err { - t.Errorf("Error loading the contents of from URL: [%s]. Received error: [%v].", outHref, err) - /////////// RETURN - return - } - - _ = ioutil.WriteFile(outFileName, respBody, 0644) - - } - outFd, err := os.Open(outFileName) - if nil != err { - t.Errorf("Could NOT open testdata file: [%s]. Received error: [%v]", outFileName, err) - /////// RETURN - return - } - defer outFd.Close() - - out := bufio.NewReaderSize(outFd, 1024) - - for { - - vocS, err := voc.ReadString('\n') - if nil != err { - /////// BREAK - break - } - - vocS = strings.Trim(vocS, "\n\r\t ") - - expected, err := out.ReadString('\n') - if nil != err { - t.Errorf("Received unexpected error when trying to read a line from [%s]. Received error: [%v]", outFileName, err) - /////// BREAK - break - - } - - expected = strings.Trim(expected, "\n\r\t ") - - actual := StemString(vocS) - if expected != actual { - t.Errorf("Input: [%s] -> Actual: [%s]. Expected: [%s]", vocS, actual, expected) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_without_lower_casing_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_without_lower_casing_test.go deleted file mode 100644 index b8b7035d..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_stem_without_lower_casing_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStemWithoutLowerCasing(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 3) - - tests[i].S = []rune("controll") - tests[i].Expected = []rune("control") - i++ - - tests[i].S = []rune("roll") - tests[i].Expected = []rune("roll") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = StemWithoutLowerCasing(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling StemWithoutLowerCasing() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1a_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1a_test.go deleted file mode 100644 index 6fa8445d..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1a_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep1a(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 12) - - tests[i].S = []rune("caresses") - tests[i].Expected = []rune("caress") - i++ - - tests[i].S = []rune("ponies") - tests[i].Expected = []rune("poni") - i++ - - tests[i].S = []rune("ties") - tests[i].Expected = []rune("ti") - i++ - - tests[i].S = []rune("caress") - tests[i].Expected = []rune("caress") - i++ - - tests[i].S = []rune("cats") - tests[i].Expected = []rune("cat") - i++ - - for _, datum := range tests { - for i = 0; i < len(datum.S); i++ { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step1a(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step1a() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } // for - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1b_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1b_test.go deleted file mode 100644 index 31b4c08d..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1b_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep1b(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 17) - - tests[i].S = []rune("feed") - tests[i].Expected = []rune("feed") - i++ - - tests[i].S = []rune("agreed") - tests[i].Expected = []rune("agree") - i++ - - tests[i].S = []rune("plastered") - tests[i].Expected = []rune("plaster") - i++ - - tests[i].S = []rune("bled") - tests[i].Expected = []rune("bled") - i++ - - tests[i].S = []rune("motoring") - tests[i].Expected = []rune("motor") - i++ - - tests[i].S = []rune("sing") - tests[i].Expected = []rune("sing") - i++ - - tests[i].S = []rune("conflated") - tests[i].Expected = []rune("conflate") - i++ - - tests[i].S = []rune("troubled") - tests[i].Expected = []rune("trouble") - i++ - - tests[i].S = []rune("sized") - tests[i].Expected = []rune("size") - i++ - - tests[i].S = []rune("hopping") - tests[i].Expected = []rune("hop") - i++ - - tests[i].S = []rune("tanned") - tests[i].Expected = []rune("tan") - i++ - - tests[i].S = []rune("falling") - tests[i].Expected = []rune("fall") - i++ - - tests[i].S = []rune("hissing") - tests[i].Expected = []rune("hiss") - i++ - - tests[i].S = []rune("fizzed") - tests[i].Expected = []rune("fizz") - i++ - - tests[i].S = []rune("failing") - tests[i].Expected = []rune("fail") - i++ - - tests[i].S = []rune("filing") - tests[i].Expected = []rune("file") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step1b(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step1b() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1c_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1c_test.go deleted file mode 100644 index 7dc864ad..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step1c_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep1c(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 17) - - tests[i].S = []rune("happy") - tests[i].Expected = []rune("happi") - i++ - - tests[i].S = []rune("sky") - tests[i].Expected = []rune("sky") - i++ - - tests[i].S = []rune("apology") - tests[i].Expected = []rune("apologi") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step1c(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step1c() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step2_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step2_test.go deleted file mode 100644 index f8258f65..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step2_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep2(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 22) - - tests[i].S = []rune("relational") - tests[i].Expected = []rune("relate") - i++ - - tests[i].S = []rune("conditional") - tests[i].Expected = []rune("condition") - i++ - - tests[i].S = []rune("rational") - tests[i].Expected = []rune("rational") - i++ - - tests[i].S = []rune("valenci") - tests[i].Expected = []rune("valence") - i++ - - tests[i].S = []rune("hesitanci") - tests[i].Expected = []rune("hesitance") - i++ - - tests[i].S = []rune("digitizer") - tests[i].Expected = []rune("digitize") - i++ - - tests[i].S = []rune("conformabli") - tests[i].Expected = []rune("conformable") - i++ - - tests[i].S = []rune("radicalli") - tests[i].Expected = []rune("radical") - i++ - - tests[i].S = []rune("differentli") - tests[i].Expected = []rune("different") - i++ - - tests[i].S = []rune("vileli") - tests[i].Expected = []rune("vile") - i++ - - tests[i].S = []rune("analogousli") - tests[i].Expected = []rune("analogous") - i++ - - tests[i].S = []rune("vietnamization") - tests[i].Expected = []rune("vietnamize") - i++ - - tests[i].S = []rune("predication") - tests[i].Expected = []rune("predicate") - i++ - - tests[i].S = []rune("operator") - tests[i].Expected = []rune("operate") - i++ - - tests[i].S = []rune("feudalism") - tests[i].Expected = []rune("feudal") - i++ - - tests[i].S = []rune("decisiveness") - tests[i].Expected = []rune("decisive") - i++ - - tests[i].S = []rune("hopefulness") - tests[i].Expected = []rune("hopeful") - i++ - - tests[i].S = []rune("callousness") - tests[i].Expected = []rune("callous") - i++ - - tests[i].S = []rune("formaliti") - tests[i].Expected = []rune("formal") - i++ - - tests[i].S = []rune("sensitiviti") - tests[i].Expected = []rune("sensitive") - i++ - - tests[i].S = []rune("sensibiliti") - tests[i].Expected = []rune("sensible") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step2(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step2() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step3_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step3_test.go deleted file mode 100644 index d2a5e9aa..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step3_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep3(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 22) - - tests[i].S = []rune("triplicate") - tests[i].Expected = []rune("triplic") - i++ - - tests[i].S = []rune("formative") - tests[i].Expected = []rune("form") - i++ - - tests[i].S = []rune("formalize") - tests[i].Expected = []rune("formal") - i++ - - tests[i].S = []rune("electriciti") - tests[i].Expected = []rune("electric") - i++ - - tests[i].S = []rune("electrical") - tests[i].Expected = []rune("electric") - i++ - - tests[i].S = []rune("hopeful") - tests[i].Expected = []rune("hope") - i++ - - tests[i].S = []rune("goodness") - tests[i].Expected = []rune("good") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step3(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step3() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step4_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step4_test.go deleted file mode 100644 index 9154490c..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step4_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep4(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 20) - - tests[i].S = []rune("revival") - tests[i].Expected = []rune("reviv") - i++ - - tests[i].S = []rune("allowance") - tests[i].Expected = []rune("allow") - i++ - - tests[i].S = []rune("inference") - tests[i].Expected = []rune("infer") - i++ - - tests[i].S = []rune("airliner") - tests[i].Expected = []rune("airlin") - i++ - - tests[i].S = []rune("gyroscopic") - tests[i].Expected = []rune("gyroscop") - i++ - - tests[i].S = []rune("adjustable") - tests[i].Expected = []rune("adjust") - i++ - - tests[i].S = []rune("defensible") - tests[i].Expected = []rune("defens") - i++ - - tests[i].S = []rune("irritant") - tests[i].Expected = []rune("irrit") - i++ - - tests[i].S = []rune("replacement") - tests[i].Expected = []rune("replac") - i++ - - tests[i].S = []rune("adjustment") - tests[i].Expected = []rune("adjust") - i++ - - tests[i].S = []rune("dependent") - tests[i].Expected = []rune("depend") - i++ - - tests[i].S = []rune("adoption") - tests[i].Expected = []rune("adopt") - i++ - - tests[i].S = []rune("homologou") - tests[i].Expected = []rune("homolog") - i++ - - tests[i].S = []rune("communism") - tests[i].Expected = []rune("commun") - i++ - - tests[i].S = []rune("activate") - tests[i].Expected = []rune("activ") - i++ - - tests[i].S = []rune("angulariti") - tests[i].Expected = []rune("angular") - i++ - - tests[i].S = []rune("homologous") - tests[i].Expected = []rune("homolog") - i++ - - tests[i].S = []rune("effective") - tests[i].Expected = []rune("effect") - i++ - - tests[i].S = []rune("bowdlerize") - tests[i].Expected = []rune("bowdler") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step4(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step4() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5a_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5a_test.go deleted file mode 100644 index 65245277..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5a_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep5a(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 3) - - tests[i].S = []rune("probate") - tests[i].Expected = []rune("probat") - i++ - - tests[i].S = []rune("rate") - tests[i].Expected = []rune("rate") - i++ - - tests[i].S = []rune("cease") - tests[i].Expected = []rune("ceas") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step5a(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step5a() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5b_test.go b/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5b_test.go deleted file mode 100644 index 233c37e8..00000000 --- a/vendor/github.com/blevesearch/go-porterstemmer/porterstemmer_step5b_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package porterstemmer - -import ( - "testing" -) - -func TestStep5b(t *testing.T) { - - i := 0 - - tests := make([]struct { - S []rune - Expected []rune - }, 3) - - tests[i].S = []rune("controll") - tests[i].Expected = []rune("control") - i++ - - tests[i].S = []rune("roll") - tests[i].Expected = []rune("roll") - i++ - - for _, datum := range tests { - - actual := make([]rune, len(datum.S)) - copy(actual, datum.S) - - actual = step5b(actual) - - lenActual := len(actual) - lenExpected := len(datum.Expected) - - equal := true - if 0 == lenActual && 0 == lenExpected { - equal = true - } else if lenActual != lenExpected { - equal = false - } else if actual[0] != datum.Expected[0] { - equal = false - } else if actual[lenActual-1] != datum.Expected[lenExpected-1] { - equal = false - } else { - for j := 0; j < lenActual; j++ { - - if actual[j] != datum.Expected[j] { - equal = false - } - } - } - - if !equal { - t.Errorf("Did NOT get what was expected for calling step5b() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual)) - } - } -} diff --git a/vendor/github.com/blevesearch/segment/export_test.go b/vendor/github.com/blevesearch/segment/export_test.go deleted file mode 100644 index ae8bddba..00000000 --- a/vendor/github.com/blevesearch/segment/export_test.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package segment - -// Exported for testing only. -import ( - "unicode/utf8" -) - -func (s *Segmenter) MaxTokenSize(n int) { - if n < utf8.UTFMax || n > 1e9 { - panic("bad max token size") - } - if n < len(s.buf) { - s.buf = make([]byte, n) - } - s.maxTokenSize = n -} diff --git a/vendor/github.com/blevesearch/segment/ragel/unicode2ragel.rb b/vendor/github.com/blevesearch/segment/ragel/unicode2ragel.rb deleted file mode 100644 index 35e09d77..00000000 --- a/vendor/github.com/blevesearch/segment/ragel/unicode2ragel.rb +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env ruby -# -# This scripted has been updated to accept more command-line arguments: -# -# -u, --url URL to process -# -m, --machine Machine name -# -p, --properties Properties to add to the machine -# -o, --output Write output to file -# -# Updated by: Marty Schoch -# -# This script uses the unicode spec to generate a Ragel state machine -# that recognizes unicode alphanumeric characters. It generates 5 -# character classes: uupper, ulower, ualpha, udigit, and ualnum. -# Currently supported encodings are UTF-8 [default] and UCS-4. -# -# Usage: unicode2ragel.rb [options] -# -e, --encoding [ucs4 | utf8] Data encoding -# -h, --help Show this message -# -# This script was originally written as part of the Ferret search -# engine library. -# -# Author: Rakan El-Khalil - -require 'optparse' -require 'open-uri' - -ENCODINGS = [ :utf8, :ucs4 ] -ALPHTYPES = { :utf8 => "unsigned char", :ucs4 => "unsigned int" } -DEFAULT_CHART_URL = "http://www.unicode.org/Public/5.1.0/ucd/DerivedCoreProperties.txt" -DEFAULT_MACHINE_NAME= "WChar" - -### -# Display vars & default option - -TOTAL_WIDTH = 80 -RANGE_WIDTH = 23 -@encoding = :utf8 -@chart_url = DEFAULT_CHART_URL -machine_name = DEFAULT_MACHINE_NAME -properties = [] -@output = $stdout - -### -# Option parsing - -cli_opts = OptionParser.new do |opts| - opts.on("-e", "--encoding [ucs4 | utf8]", "Data encoding") do |o| - @encoding = o.downcase.to_sym - end - opts.on("-h", "--help", "Show this message") do - puts opts - exit - end - opts.on("-u", "--url URL", "URL to process") do |o| - @chart_url = o - end - opts.on("-m", "--machine MACHINE_NAME", "Machine name") do |o| - machine_name = o - end - opts.on("-p", "--properties x,y,z", Array, "Properties to add to machine") do |o| - properties = o - end - opts.on("-o", "--output FILE", "output file") do |o| - @output = File.new(o, "w+") - end -end - -cli_opts.parse(ARGV) -unless ENCODINGS.member? @encoding - puts "Invalid encoding: #{@encoding}" - puts cli_opts - exit -end - -## -# Downloads the document at url and yields every alpha line's hex -# range and description. - -def each_alpha( url, property ) - open( url ) do |file| - file.each_line do |line| - next if line =~ /^#/; - next if line !~ /; #{property} #/; - - range, description = line.split(/;/) - range.strip! - description.gsub!(/.*#/, '').strip! - - if range =~ /\.\./ - start, stop = range.split '..' - else start = stop = range - end - - yield start.hex .. stop.hex, description - end - end -end - -### -# Formats to hex at minimum width - -def to_hex( n ) - r = "%0X" % n - r = "0#{r}" unless (r.length % 2).zero? - r -end - -### -# UCS4 is just a straight hex conversion of the unicode codepoint. - -def to_ucs4( range ) - rangestr = "0x" + to_hex(range.begin) - rangestr << "..0x" + to_hex(range.end) if range.begin != range.end - [ rangestr ] -end - -## -# 0x00 - 0x7f -> 0zzzzzzz[7] -# 0x80 - 0x7ff -> 110yyyyy[5] 10zzzzzz[6] -# 0x800 - 0xffff -> 1110xxxx[4] 10yyyyyy[6] 10zzzzzz[6] -# 0x010000 - 0x10ffff -> 11110www[3] 10xxxxxx[6] 10yyyyyy[6] 10zzzzzz[6] - -UTF8_BOUNDARIES = [0x7f, 0x7ff, 0xffff, 0x10ffff] - -def to_utf8_enc( n ) - r = 0 - if n <= 0x7f - r = n - elsif n <= 0x7ff - y = 0xc0 | (n >> 6) - z = 0x80 | (n & 0x3f) - r = y << 8 | z - elsif n <= 0xffff - x = 0xe0 | (n >> 12) - y = 0x80 | (n >> 6) & 0x3f - z = 0x80 | n & 0x3f - r = x << 16 | y << 8 | z - elsif n <= 0x10ffff - w = 0xf0 | (n >> 18) - x = 0x80 | (n >> 12) & 0x3f - y = 0x80 | (n >> 6) & 0x3f - z = 0x80 | n & 0x3f - r = w << 24 | x << 16 | y << 8 | z - end - - to_hex(r) -end - -def from_utf8_enc( n ) - n = n.hex - r = 0 - if n <= 0x7f - r = n - elsif n <= 0xdfff - y = (n >> 8) & 0x1f - z = n & 0x3f - r = y << 6 | z - elsif n <= 0xefffff - x = (n >> 16) & 0x0f - y = (n >> 8) & 0x3f - z = n & 0x3f - r = x << 10 | y << 6 | z - elsif n <= 0xf7ffffff - w = (n >> 24) & 0x07 - x = (n >> 16) & 0x3f - y = (n >> 8) & 0x3f - z = n & 0x3f - r = w << 18 | x << 12 | y << 6 | z - end - r -end - -### -# Given a range, splits it up into ranges that can be continuously -# encoded into utf8. Eg: 0x00 .. 0xff => [0x00..0x7f, 0x80..0xff] -# This is not strictly needed since the current [5.1] unicode standard -# doesn't have ranges that straddle utf8 boundaries. This is included -# for completeness as there is no telling if that will ever change. - -def utf8_ranges( range ) - ranges = [] - UTF8_BOUNDARIES.each do |max| - if range.begin <= max - return ranges << range if range.end <= max - - ranges << range.begin .. max - range = (max + 1) .. range.end - end - end - ranges -end - -def build_range( start, stop ) - size = start.size/2 - left = size - 1 - return [""] if size < 1 - - a = start[0..1] - b = stop[0..1] - - ### - # Shared prefix - - if a == b - return build_range(start[2..-1], stop[2..-1]).map do |elt| - "0x#{a} " + elt - end - end - - ### - # Unshared prefix, end of run - - return ["0x#{a}..0x#{b} "] if left.zero? - - ### - # Unshared prefix, not end of run - # Range can be 0x123456..0x56789A - # Which is equivalent to: - # 0x123456 .. 0x12FFFF - # 0x130000 .. 0x55FFFF - # 0x560000 .. 0x56789A - - ret = [] - ret << build_range(start, a + "FF" * left) - - ### - # Only generate middle range if need be. - - if a.hex+1 != b.hex - max = to_hex(b.hex - 1) - max = "FF" if b == "FF" - ret << "0x#{to_hex(a.hex+1)}..0x#{max} " + "0x00..0xFF " * left - end - - ### - # Don't generate last range if it is covered by first range - - ret << build_range(b + "00" * left, stop) unless b == "FF" - ret.flatten! -end - -def to_utf8( range ) - utf8_ranges( range ).map do |r| - build_range to_utf8_enc(r.begin), to_utf8_enc(r.end) - end.flatten! -end - -## -# Perform a 3-way comparison of the number of codepoints advertised by -# the unicode spec for the given range, the originally parsed range, -# and the resulting utf8 encoded range. - -def count_codepoints( code ) - code.split(' ').inject(1) do |acc, elt| - if elt =~ /0x(.+)\.\.0x(.+)/ - if @encoding == :utf8 - acc * (from_utf8_enc($2) - from_utf8_enc($1) + 1) - else - acc * ($2.hex - $1.hex + 1) - end - else - acc - end - end -end - -def is_valid?( range, desc, codes ) - spec_count = 1 - spec_count = $1.to_i if desc =~ /\[(\d+)\]/ - range_count = range.end - range.begin + 1 - - sum = codes.inject(0) { |acc, elt| acc + count_codepoints(elt) } - sum == spec_count and sum == range_count -end - -## -# Generate the state maching to stdout - -def generate_machine( name, property ) - pipe = " " - @output.puts " #{name} = " - each_alpha( @chart_url, property ) do |range, desc| - - codes = (@encoding == :ucs4) ? to_ucs4(range) : to_utf8(range) - - raise "Invalid encoding of range #{range}: #{codes.inspect}" unless - is_valid? range, desc, codes - - range_width = codes.map { |a| a.size }.max - range_width = RANGE_WIDTH if range_width < RANGE_WIDTH - - desc_width = TOTAL_WIDTH - RANGE_WIDTH - 11 - desc_width -= (range_width - RANGE_WIDTH) if range_width > RANGE_WIDTH - - if desc.size > desc_width - desc = desc[0..desc_width - 4] + "..." - end - - codes.each_with_index do |r, idx| - desc = "" unless idx.zero? - code = "%-#{range_width}s" % r - @output.puts " #{pipe} #{code} ##{desc}" - pipe = "|" - end - end - @output.puts " ;" - @output.puts "" -end - -@output.puts < - ; - - LF = - 0x0A #Cc - ; - - Newline = - 0x0B..0x0C #Cc [2] .. - | 0xC2 0x85 #Cc - | 0xE2 0x80 0xA8 #Zl LINE SEPARATOR - | 0xE2 0x80 0xA9 #Zp PARAGRAPH SEPARATOR - ; - - Extend = - 0xCC 0x80..0xFF #Mn [112] COMBINING GRAVE ACCENT..COMBINING ... - | 0xCD 0x00..0xAF # - | 0xD2 0x83..0x87 #Mn [5] COMBINING CYRILLIC TITLO..COMBININ... - | 0xD2 0x88..0x89 #Me [2] COMBINING CYRILLIC HUNDRED THOUSAN... - | 0xD6 0x91..0xBD #Mn [45] HEBREW ACCENT ETNAHTA..HEBREW POIN... - | 0xD6 0xBF #Mn HEBREW POINT RAFE - | 0xD7 0x81..0x82 #Mn [2] HEBREW POINT SHIN DOT..HEBREW POIN... - | 0xD7 0x84..0x85 #Mn [2] HEBREW MARK UPPER DOT..HEBREW MARK... - | 0xD7 0x87 #Mn HEBREW POINT QAMATS QATAN - | 0xD8 0x90..0x9A #Mn [11] ARABIC SIGN SALLALLAHOU ALAYHE WAS... - | 0xD9 0x8B..0x9F #Mn [21] ARABIC FATHATAN..ARABIC WAVY HAMZA... - | 0xD9 0xB0 #Mn ARABIC LETTER SUPERSCRIPT ALEF - | 0xDB 0x96..0x9C #Mn [7] ARABIC SMALL HIGH LIGATURE SAD WIT... - | 0xDB 0x9F..0xA4 #Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..AR... - | 0xDB 0xA7..0xA8 #Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMAL... - | 0xDB 0xAA..0xAD #Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARAB... - | 0xDC 0x91 #Mn SYRIAC LETTER SUPERSCRIPT ALAPH - | 0xDC 0xB0..0xFF #Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH - | 0xDD 0x00..0x8A # - | 0xDE 0xA6..0xB0 #Mn [11] THAANA ABAFILI..THAANA SUKUN - | 0xDF 0xAB..0xB3 #Mn [9] NKO COMBINING SHORT HIGH TONE..NKO... - | 0xE0 0xA0 0x96..0x99 #Mn [4] SAMARITAN MARK IN..SAMARITAN MARK ... - | 0xE0 0xA0 0x9B..0xA3 #Mn [9] SAMARITAN MARK EPENTHETIC YUT..SAM... - | 0xE0 0xA0 0xA5..0xA7 #Mn [3] SAMARITAN VOWEL SIGN SHORT A..SAMA... - | 0xE0 0xA0 0xA9..0xAD #Mn [5] SAMARITAN VOWEL SIGN LONG I..SAMAR... - | 0xE0 0xA1 0x99..0x9B #Mn [3] MANDAIC AFFRICATION MARK..MANDAIC ... - | 0xE0 0xA3 0xA3..0xFF #Mn [32] ARABIC TURNED DAMMA BELOW..DEVANAG... - | 0xE0 0xA4 0x00..0x82 # - | 0xE0 0xA4 0x83 #Mc DEVANAGARI SIGN VISARGA - | 0xE0 0xA4 0xBA #Mn DEVANAGARI VOWEL SIGN OE - | 0xE0 0xA4 0xBB #Mc DEVANAGARI VOWEL SIGN OOE - | 0xE0 0xA4 0xBC #Mn DEVANAGARI SIGN NUKTA - | 0xE0 0xA4 0xBE..0xFF #Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGA... - | 0xE0 0xA5 0x00..0x80 # - | 0xE0 0xA5 0x81..0x88 #Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGAR... - | 0xE0 0xA5 0x89..0x8C #Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DE... - | 0xE0 0xA5 0x8D #Mn DEVANAGARI SIGN VIRAMA - | 0xE0 0xA5 0x8E..0x8F #Mc [2] DEVANAGARI VOWEL SIGN PRISHTHAMATR... - | 0xE0 0xA5 0x91..0x97 #Mn [7] DEVANAGARI STRESS SIGN UDATTA..DEV... - | 0xE0 0xA5 0xA2..0xA3 #Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..D... - | 0xE0 0xA6 0x81 #Mn BENGALI SIGN CANDRABINDU - | 0xE0 0xA6 0x82..0x83 #Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIG... - | 0xE0 0xA6 0xBC #Mn BENGALI SIGN NUKTA - | 0xE0 0xA6 0xBE..0xFF #Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOW... - | 0xE0 0xA7 0x00..0x80 # - | 0xE0 0xA7 0x81..0x84 #Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWE... - | 0xE0 0xA7 0x87..0x88 #Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWE... - | 0xE0 0xA7 0x8B..0x8C #Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWE... - | 0xE0 0xA7 0x8D #Mn BENGALI SIGN VIRAMA - | 0xE0 0xA7 0x97 #Mc BENGALI AU LENGTH MARK - | 0xE0 0xA7 0xA2..0xA3 #Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENG... - | 0xE0 0xA8 0x81..0x82 #Mn [2] GURMUKHI SIGN ADAK BINDI..GURMUKHI... - | 0xE0 0xA8 0x83 #Mc GURMUKHI SIGN VISARGA - | 0xE0 0xA8 0xBC #Mn GURMUKHI SIGN NUKTA - | 0xE0 0xA8 0xBE..0xFF #Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI V... - | 0xE0 0xA9 0x00..0x80 # - | 0xE0 0xA9 0x81..0x82 #Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VO... - | 0xE0 0xA9 0x87..0x88 #Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI V... - | 0xE0 0xA9 0x8B..0x8D #Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI S... - | 0xE0 0xA9 0x91 #Mn GURMUKHI SIGN UDAAT - | 0xE0 0xA9 0xB0..0xB1 #Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK - | 0xE0 0xA9 0xB5 #Mn GURMUKHI SIGN YAKASH - | 0xE0 0xAA 0x81..0x82 #Mn [2] GUJARATI SIGN CANDRABINDU..GUJARAT... - | 0xE0 0xAA 0x83 #Mc GUJARATI SIGN VISARGA - | 0xE0 0xAA 0xBC #Mn GUJARATI SIGN NUKTA - | 0xE0 0xAA 0xBE..0xFF #Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI V... - | 0xE0 0xAB 0x00..0x80 # - | 0xE0 0xAB 0x81..0x85 #Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VO... - | 0xE0 0xAB 0x87..0x88 #Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VO... - | 0xE0 0xAB 0x89 #Mc GUJARATI VOWEL SIGN CANDRA O - | 0xE0 0xAB 0x8B..0x8C #Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VO... - | 0xE0 0xAB 0x8D #Mn GUJARATI SIGN VIRAMA - | 0xE0 0xAB 0xA2..0xA3 #Mn [2] GUJARATI VOWEL SIGN VOCALIC L..GUJ... - | 0xE0 0xAC 0x81 #Mn ORIYA SIGN CANDRABINDU - | 0xE0 0xAC 0x82..0x83 #Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VI... - | 0xE0 0xAC 0xBC #Mn ORIYA SIGN NUKTA - | 0xE0 0xAC 0xBE #Mc ORIYA VOWEL SIGN AA - | 0xE0 0xAC 0xBF #Mn ORIYA VOWEL SIGN I - | 0xE0 0xAD 0x80 #Mc ORIYA VOWEL SIGN II - | 0xE0 0xAD 0x81..0x84 #Mn [4] ORIYA VOWEL SIGN U..ORIYA VOWEL SI... - | 0xE0 0xAD 0x87..0x88 #Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SI... - | 0xE0 0xAD 0x8B..0x8C #Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SI... - | 0xE0 0xAD 0x8D #Mn ORIYA SIGN VIRAMA - | 0xE0 0xAD 0x96 #Mn ORIYA AI LENGTH MARK - | 0xE0 0xAD 0x97 #Mc ORIYA AU LENGTH MARK - | 0xE0 0xAD 0xA2..0xA3 #Mn [2] ORIYA VOWEL SIGN VOCALIC L..ORIYA ... - | 0xE0 0xAE 0x82 #Mn TAMIL SIGN ANUSVARA - | 0xE0 0xAE 0xBE..0xBF #Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL S... - | 0xE0 0xAF 0x80 #Mn TAMIL VOWEL SIGN II - | 0xE0 0xAF 0x81..0x82 #Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SI... - | 0xE0 0xAF 0x86..0x88 #Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SI... - | 0xE0 0xAF 0x8A..0x8C #Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SI... - | 0xE0 0xAF 0x8D #Mn TAMIL SIGN VIRAMA - | 0xE0 0xAF 0x97 #Mc TAMIL AU LENGTH MARK - | 0xE0 0xB0 0x80 #Mn TELUGU SIGN COMBINING CANDRABINDU ... - | 0xE0 0xB0 0x81..0x83 #Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SI... - | 0xE0 0xB0 0xBE..0xFF #Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL... - | 0xE0 0xB1 0x00..0x80 # - | 0xE0 0xB1 0x81..0x84 #Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL ... - | 0xE0 0xB1 0x86..0x88 #Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL ... - | 0xE0 0xB1 0x8A..0x8D #Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN V... - | 0xE0 0xB1 0x95..0x96 #Mn [2] TELUGU LENGTH MARK..TELUGU AI LENG... - | 0xE0 0xB1 0xA2..0xA3 #Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUG... - | 0xE0 0xB2 0x81 #Mn KANNADA SIGN CANDRABINDU - | 0xE0 0xB2 0x82..0x83 #Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIG... - | 0xE0 0xB2 0xBC #Mn KANNADA SIGN NUKTA - | 0xE0 0xB2 0xBE #Mc KANNADA VOWEL SIGN AA - | 0xE0 0xB2 0xBF #Mn KANNADA VOWEL SIGN I - | 0xE0 0xB3 0x80..0x84 #Mc [5] KANNADA VOWEL SIGN II..KANNADA VOW... - | 0xE0 0xB3 0x86 #Mn KANNADA VOWEL SIGN E - | 0xE0 0xB3 0x87..0x88 #Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOW... - | 0xE0 0xB3 0x8A..0x8B #Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWE... - | 0xE0 0xB3 0x8C..0x8D #Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIG... - | 0xE0 0xB3 0x95..0x96 #Mc [2] KANNADA LENGTH MARK..KANNADA AI LE... - | 0xE0 0xB3 0xA2..0xA3 #Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANN... - | 0xE0 0xB4 0x81 #Mn MALAYALAM SIGN CANDRABINDU - | 0xE0 0xB4 0x82..0x83 #Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM... - | 0xE0 0xB4 0xBE..0xFF #Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM... - | 0xE0 0xB5 0x00..0x80 # - | 0xE0 0xB5 0x81..0x84 #Mn [4] MALAYALAM VOWEL SIGN U..MALAYALAM ... - | 0xE0 0xB5 0x86..0x88 #Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM ... - | 0xE0 0xB5 0x8A..0x8C #Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM ... - | 0xE0 0xB5 0x8D #Mn MALAYALAM SIGN VIRAMA - | 0xE0 0xB5 0x97 #Mc MALAYALAM AU LENGTH MARK - | 0xE0 0xB5 0xA2..0xA3 #Mn [2] MALAYALAM VOWEL SIGN VOCALIC L..MA... - | 0xE0 0xB6 0x82..0x83 #Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA S... - | 0xE0 0xB7 0x8A #Mn SINHALA SIGN AL-LAKUNA - | 0xE0 0xB7 0x8F..0x91 #Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SIN... - | 0xE0 0xB7 0x92..0x94 #Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA.... - | 0xE0 0xB7 0x96 #Mn SINHALA VOWEL SIGN DIGA PAA-PILLA - | 0xE0 0xB7 0x98..0x9F #Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..S... - | 0xE0 0xB7 0xB2..0xB3 #Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PIL... - | 0xE0 0xB8 0xB1 #Mn THAI CHARACTER MAI HAN-AKAT - | 0xE0 0xB8 0xB4..0xBA #Mn [7] THAI CHARACTER SARA I..THAI CHARAC... - | 0xE0 0xB9 0x87..0x8E #Mn [8] THAI CHARACTER MAITAIKHU..THAI CHA... - | 0xE0 0xBA 0xB1 #Mn LAO VOWEL SIGN MAI KAN - | 0xE0 0xBA 0xB4..0xB9 #Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU - | 0xE0 0xBA 0xBB..0xBC #Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVO... - | 0xE0 0xBB 0x88..0x8D #Mn [6] LAO TONE MAI EK..LAO NIGGAHITA - | 0xE0 0xBC 0x98..0x99 #Mn [2] TIBETAN ASTROLOGICAL SIGN -KHYUD P... - | 0xE0 0xBC 0xB5 #Mn TIBETAN MARK NGAS BZUNG NYI ZLA - | 0xE0 0xBC 0xB7 #Mn TIBETAN MARK NGAS BZUNG SGOR RTAGS - | 0xE0 0xBC 0xB9 #Mn TIBETAN MARK TSA -PHRU - | 0xE0 0xBC 0xBE..0xBF #Mc [2] TIBETAN SIGN YAR TSHES..TIBETAN SI... - | 0xE0 0xBD 0xB1..0xBE #Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIG... - | 0xE0 0xBD 0xBF #Mc TIBETAN SIGN RNAM BCAD - | 0xE0 0xBE 0x80..0x84 #Mn [5] TIBETAN VOWEL SIGN REVERSED I..TIB... - | 0xE0 0xBE 0x86..0x87 #Mn [2] TIBETAN SIGN LCI RTAGS..TIBETAN SI... - | 0xE0 0xBE 0x8D..0x97 #Mn [11] TIBETAN SUBJOINED SIGN LCE TSA CAN... - | 0xE0 0xBE 0x99..0xBC #Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBE... - | 0xE0 0xBF 0x86 #Mn TIBETAN SYMBOL PADMA GDAN - | 0xE1 0x80 0xAB..0xAC #Mc [2] MYANMAR VOWEL SIGN TALL AA..MYANMA... - | 0xE1 0x80 0xAD..0xB0 #Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWE... - | 0xE1 0x80 0xB1 #Mc MYANMAR VOWEL SIGN E - | 0xE1 0x80 0xB2..0xB7 #Mn [6] MYANMAR VOWEL SIGN AI..MYANMAR SIG... - | 0xE1 0x80 0xB8 #Mc MYANMAR SIGN VISARGA - | 0xE1 0x80 0xB9..0xBA #Mn [2] MYANMAR SIGN VIRAMA..MYANMAR SIGN ... - | 0xE1 0x80 0xBB..0xBC #Mc [2] MYANMAR CONSONANT SIGN MEDIAL YA..... - | 0xE1 0x80 0xBD..0xBE #Mn [2] MYANMAR CONSONANT SIGN MEDIAL WA..... - | 0xE1 0x81 0x96..0x97 #Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYAN... - | 0xE1 0x81 0x98..0x99 #Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYAN... - | 0xE1 0x81 0x9E..0xA0 #Mn [3] MYANMAR CONSONANT SIGN MON MEDIAL ... - | 0xE1 0x81 0xA2..0xA4 #Mc [3] MYANMAR VOWEL SIGN SGAW KAREN EU..... - | 0xE1 0x81 0xA7..0xAD #Mc [7] MYANMAR VOWEL SIGN WESTERN PWO KAR... - | 0xE1 0x81 0xB1..0xB4 #Mn [4] MYANMAR VOWEL SIGN GEBA KAREN I..M... - | 0xE1 0x82 0x82 #Mn MYANMAR CONSONANT SIGN SHAN MEDIAL WA - | 0xE1 0x82 0x83..0x84 #Mc [2] MYANMAR VOWEL SIGN SHAN AA..MYANMA... - | 0xE1 0x82 0x85..0x86 #Mn [2] MYANMAR VOWEL SIGN SHAN E ABOVE..M... - | 0xE1 0x82 0x87..0x8C #Mc [6] MYANMAR SIGN SHAN TONE-2..MYANMAR ... - | 0xE1 0x82 0x8D #Mn MYANMAR SIGN SHAN COUNCIL EMPHATIC... - | 0xE1 0x82 0x8F #Mc MYANMAR SIGN RUMAI PALAUNG TONE-5 - | 0xE1 0x82 0x9A..0x9C #Mc [3] MYANMAR SIGN KHAMTI TONE-1..MYANMA... - | 0xE1 0x82 0x9D #Mn MYANMAR VOWEL SIGN AITON AI - | 0xE1 0x8D 0x9D..0x9F #Mn [3] ETHIOPIC COMBINING GEMINATION AND ... - | 0xE1 0x9C 0x92..0x94 #Mn [3] TAGALOG VOWEL SIGN I..TAGALOG SIGN... - | 0xE1 0x9C 0xB2..0xB4 #Mn [3] HANUNOO VOWEL SIGN I..HANUNOO SIGN... - | 0xE1 0x9D 0x92..0x93 #Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SI... - | 0xE1 0x9D 0xB2..0xB3 #Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VO... - | 0xE1 0x9E 0xB4..0xB5 #Mn [2] KHMER VOWEL INHERENT AQ..KHMER VOW... - | 0xE1 0x9E 0xB6 #Mc KHMER VOWEL SIGN AA - | 0xE1 0x9E 0xB7..0xBD #Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SI... - | 0xE1 0x9E 0xBE..0xFF #Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL S... - | 0xE1 0x9F 0x00..0x85 # - | 0xE1 0x9F 0x86 #Mn KHMER SIGN NIKAHIT - | 0xE1 0x9F 0x87..0x88 #Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUU... - | 0xE1 0x9F 0x89..0x93 #Mn [11] KHMER SIGN MUUSIKATOAN..KHMER SIGN... - | 0xE1 0x9F 0x9D #Mn KHMER SIGN ATTHACAN - | 0xE1 0xA0 0x8B..0x8D #Mn [3] MONGOLIAN FREE VARIATION SELECTOR ... - | 0xE1 0xA2 0xA9 #Mn MONGOLIAN LETTER ALI GALI DAGALGA - | 0xE1 0xA4 0xA0..0xA2 #Mn [3] LIMBU VOWEL SIGN A..LIMBU VOWEL SI... - | 0xE1 0xA4 0xA3..0xA6 #Mc [4] LIMBU VOWEL SIGN EE..LIMBU VOWEL S... - | 0xE1 0xA4 0xA7..0xA8 #Mn [2] LIMBU VOWEL SIGN E..LIMBU VOWEL SI... - | 0xE1 0xA4 0xA9..0xAB #Mc [3] LIMBU SUBJOINED LETTER YA..LIMBU S... - | 0xE1 0xA4 0xB0..0xB1 #Mc [2] LIMBU SMALL LETTER KA..LIMBU SMALL... - | 0xE1 0xA4 0xB2 #Mn LIMBU SMALL LETTER ANUSVARA - | 0xE1 0xA4 0xB3..0xB8 #Mc [6] LIMBU SMALL LETTER TA..LIMBU SMALL... - | 0xE1 0xA4 0xB9..0xBB #Mn [3] LIMBU SIGN MUKPHRENG..LIMBU SIGN SA-I - | 0xE1 0xA8 0x97..0x98 #Mn [2] BUGINESE VOWEL SIGN I..BUGINESE VO... - | 0xE1 0xA8 0x99..0x9A #Mc [2] BUGINESE VOWEL SIGN E..BUGINESE VO... - | 0xE1 0xA8 0x9B #Mn BUGINESE VOWEL SIGN AE - | 0xE1 0xA9 0x95 #Mc TAI THAM CONSONANT SIGN MEDIAL RA - | 0xE1 0xA9 0x96 #Mn TAI THAM CONSONANT SIGN MEDIAL LA - | 0xE1 0xA9 0x97 #Mc TAI THAM CONSONANT SIGN LA TANG LAI - | 0xE1 0xA9 0x98..0x9E #Mn [7] TAI THAM SIGN MAI KANG LAI..TAI TH... - | 0xE1 0xA9 0xA0 #Mn TAI THAM SIGN SAKOT - | 0xE1 0xA9 0xA1 #Mc TAI THAM VOWEL SIGN A - | 0xE1 0xA9 0xA2 #Mn TAI THAM VOWEL SIGN MAI SAT - | 0xE1 0xA9 0xA3..0xA4 #Mc [2] TAI THAM VOWEL SIGN AA..TAI THAM V... - | 0xE1 0xA9 0xA5..0xAC #Mn [8] TAI THAM VOWEL SIGN I..TAI THAM VO... - | 0xE1 0xA9 0xAD..0xB2 #Mc [6] TAI THAM VOWEL SIGN OY..TAI THAM V... - | 0xE1 0xA9 0xB3..0xBC #Mn [10] TAI THAM VOWEL SIGN OA ABOVE..TAI ... - | 0xE1 0xA9 0xBF #Mn TAI THAM COMBINING CRYPTOGRAMMIC DOT - | 0xE1 0xAA 0xB0..0xBD #Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCEN... - | 0xE1 0xAA 0xBE #Me COMBINING PARENTHESES OVERLAY - | 0xE1 0xAC 0x80..0x83 #Mn [4] BALINESE SIGN ULU RICEM..BALINESE ... - | 0xE1 0xAC 0x84 #Mc BALINESE SIGN BISAH - | 0xE1 0xAC 0xB4 #Mn BALINESE SIGN REREKAN - | 0xE1 0xAC 0xB5 #Mc BALINESE VOWEL SIGN TEDUNG - | 0xE1 0xAC 0xB6..0xBA #Mn [5] BALINESE VOWEL SIGN ULU..BALINESE ... - | 0xE1 0xAC 0xBB #Mc BALINESE VOWEL SIGN RA REPA TEDUNG - | 0xE1 0xAC 0xBC #Mn BALINESE VOWEL SIGN LA LENGA - | 0xE1 0xAC 0xBD..0xFF #Mc [5] BALINESE VOWEL SIGN LA LENGA TEDUN... - | 0xE1 0xAD 0x00..0x81 # - | 0xE1 0xAD 0x82 #Mn BALINESE VOWEL SIGN PEPET - | 0xE1 0xAD 0x83..0x84 #Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..... - | 0xE1 0xAD 0xAB..0xB3 #Mn [9] BALINESE MUSICAL SYMBOL COMBINING ... - | 0xE1 0xAE 0x80..0x81 #Mn [2] SUNDANESE SIGN PANYECEK..SUNDANESE... - | 0xE1 0xAE 0x82 #Mc SUNDANESE SIGN PANGWISAD - | 0xE1 0xAE 0xA1 #Mc SUNDANESE CONSONANT SIGN PAMINGKAL - | 0xE1 0xAE 0xA2..0xA5 #Mn [4] SUNDANESE CONSONANT SIGN PANYAKRA.... - | 0xE1 0xAE 0xA6..0xA7 #Mc [2] SUNDANESE VOWEL SIGN PANAELAENG..S... - | 0xE1 0xAE 0xA8..0xA9 #Mn [2] SUNDANESE VOWEL SIGN PAMEPET..SUND... - | 0xE1 0xAE 0xAA #Mc SUNDANESE SIGN PAMAAEH - | 0xE1 0xAE 0xAB..0xAD #Mn [3] SUNDANESE SIGN VIRAMA..SUNDANESE C... - | 0xE1 0xAF 0xA6 #Mn BATAK SIGN TOMPI - | 0xE1 0xAF 0xA7 #Mc BATAK VOWEL SIGN E - | 0xE1 0xAF 0xA8..0xA9 #Mn [2] BATAK VOWEL SIGN PAKPAK E..BATAK V... - | 0xE1 0xAF 0xAA..0xAC #Mc [3] BATAK VOWEL SIGN I..BATAK VOWEL SI... - | 0xE1 0xAF 0xAD #Mn BATAK VOWEL SIGN KARO O - | 0xE1 0xAF 0xAE #Mc BATAK VOWEL SIGN U - | 0xE1 0xAF 0xAF..0xB1 #Mn [3] BATAK VOWEL SIGN U FOR SIMALUNGUN ... - | 0xE1 0xAF 0xB2..0xB3 #Mc [2] BATAK PANGOLAT..BATAK PANONGONAN - | 0xE1 0xB0 0xA4..0xAB #Mc [8] LEPCHA SUBJOINED LETTER YA..LEPCHA... - | 0xE1 0xB0 0xAC..0xB3 #Mn [8] LEPCHA VOWEL SIGN E..LEPCHA CONSON... - | 0xE1 0xB0 0xB4..0xB5 #Mc [2] LEPCHA CONSONANT SIGN NYIN-DO..LEP... - | 0xE1 0xB0 0xB6..0xB7 #Mn [2] LEPCHA SIGN RAN..LEPCHA SIGN NUKTA - | 0xE1 0xB3 0x90..0x92 #Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PR... - | 0xE1 0xB3 0x94..0xA0 #Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVAR... - | 0xE1 0xB3 0xA1 #Mc VEDIC TONE ATHARVAVEDIC INDEPENDEN... - | 0xE1 0xB3 0xA2..0xA8 #Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC ... - | 0xE1 0xB3 0xAD #Mn VEDIC SIGN TIRYAK - | 0xE1 0xB3 0xB2..0xB3 #Mc [2] VEDIC SIGN ARDHAVISARGA..VEDIC SIG... - | 0xE1 0xB3 0xB4 #Mn VEDIC TONE CANDRA ABOVE - | 0xE1 0xB3 0xB8..0xB9 #Mn [2] VEDIC TONE RING ABOVE..VEDIC TONE ... - | 0xE1 0xB7 0x80..0xB5 #Mn [54] COMBINING DOTTED GRAVE ACCENT..COM... - | 0xE1 0xB7 0xBC..0xBF #Mn [4] COMBINING DOUBLE INVERTED BREVE BE... - | 0xE2 0x80 0x8C..0x8D #Cf [2] ZERO WIDTH NON-JOINER..ZERO WIDTH ... - | 0xE2 0x83 0x90..0x9C #Mn [13] COMBINING LEFT HARPOON ABOVE..COMB... - | 0xE2 0x83 0x9D..0xA0 #Me [4] COMBINING ENCLOSING CIRCLE..COMBIN... - | 0xE2 0x83 0xA1 #Mn COMBINING LEFT RIGHT ARROW ABOVE - | 0xE2 0x83 0xA2..0xA4 #Me [3] COMBINING ENCLOSING SCREEN..COMBIN... - | 0xE2 0x83 0xA5..0xB0 #Mn [12] COMBINING REVERSE SOLIDUS OVERLAY.... - | 0xE2 0xB3 0xAF..0xB1 #Mn [3] COPTIC COMBINING NI ABOVE..COPTIC ... - | 0xE2 0xB5 0xBF #Mn TIFINAGH CONSONANT JOINER - | 0xE2 0xB7 0xA0..0xBF #Mn [32] COMBINING CYRILLIC LETTER BE..COMB... - | 0xE3 0x80 0xAA..0xAD #Mn [4] IDEOGRAPHIC LEVEL TONE MARK..IDEOG... - | 0xE3 0x80 0xAE..0xAF #Mc [2] HANGUL SINGLE DOT TONE MARK..HANGU... - | 0xE3 0x82 0x99..0x9A #Mn [2] COMBINING KATAKANA-HIRAGANA VOICED... - | 0xEA 0x99 0xAF #Mn COMBINING CYRILLIC VZMET - | 0xEA 0x99 0xB0..0xB2 #Me [3] COMBINING CYRILLIC TEN MILLIONS SI... - | 0xEA 0x99 0xB4..0xBD #Mn [10] COMBINING CYRILLIC LETTER UKRAINIA... - | 0xEA 0x9A 0x9E..0x9F #Mn [2] COMBINING CYRILLIC LETTER EF..COMB... - | 0xEA 0x9B 0xB0..0xB1 #Mn [2] BAMUM COMBINING MARK KOQNDON..BAMU... - | 0xEA 0xA0 0x82 #Mn SYLOTI NAGRI SIGN DVISVARA - | 0xEA 0xA0 0x86 #Mn SYLOTI NAGRI SIGN HASANTA - | 0xEA 0xA0 0x8B #Mn SYLOTI NAGRI SIGN ANUSVARA - | 0xEA 0xA0 0xA3..0xA4 #Mc [2] SYLOTI NAGRI VOWEL SIGN A..SYLOTI ... - | 0xEA 0xA0 0xA5..0xA6 #Mn [2] SYLOTI NAGRI VOWEL SIGN U..SYLOTI ... - | 0xEA 0xA0 0xA7 #Mc SYLOTI NAGRI VOWEL SIGN OO - | 0xEA 0xA2 0x80..0x81 #Mc [2] SAURASHTRA SIGN ANUSVARA..SAURASHT... - | 0xEA 0xA2 0xB4..0xFF #Mc [16] SAURASHTRA CONSONANT SIGN HAARU..S... - | 0xEA 0xA3 0x00..0x83 # - | 0xEA 0xA3 0x84 #Mn SAURASHTRA SIGN VIRAMA - | 0xEA 0xA3 0xA0..0xB1 #Mn [18] COMBINING DEVANAGARI DIGIT ZERO..C... - | 0xEA 0xA4 0xA6..0xAD #Mn [8] KAYAH LI VOWEL UE..KAYAH LI TONE C... - | 0xEA 0xA5 0x87..0x91 #Mn [11] REJANG VOWEL SIGN I..REJANG CONSON... - | 0xEA 0xA5 0x92..0x93 #Mc [2] REJANG CONSONANT SIGN H..REJANG VI... - | 0xEA 0xA6 0x80..0x82 #Mn [3] JAVANESE SIGN PANYANGGA..JAVANESE ... - | 0xEA 0xA6 0x83 #Mc JAVANESE SIGN WIGNYAN - | 0xEA 0xA6 0xB3 #Mn JAVANESE SIGN CECAK TELU - | 0xEA 0xA6 0xB4..0xB5 #Mc [2] JAVANESE VOWEL SIGN TARUNG..JAVANE... - | 0xEA 0xA6 0xB6..0xB9 #Mn [4] JAVANESE VOWEL SIGN WULU..JAVANESE... - | 0xEA 0xA6 0xBA..0xBB #Mc [2] JAVANESE VOWEL SIGN TALING..JAVANE... - | 0xEA 0xA6 0xBC #Mn JAVANESE VOWEL SIGN PEPET - | 0xEA 0xA6 0xBD..0xFF #Mc [4] JAVANESE CONSONANT SIGN KERET..JAV... - | 0xEA 0xA7 0x00..0x80 # - | 0xEA 0xA7 0xA5 #Mn MYANMAR SIGN SHAN SAW - | 0xEA 0xA8 0xA9..0xAE #Mn [6] CHAM VOWEL SIGN AA..CHAM VOWEL SIG... - | 0xEA 0xA8 0xAF..0xB0 #Mc [2] CHAM VOWEL SIGN O..CHAM VOWEL SIGN AI - | 0xEA 0xA8 0xB1..0xB2 #Mn [2] CHAM VOWEL SIGN AU..CHAM VOWEL SIG... - | 0xEA 0xA8 0xB3..0xB4 #Mc [2] CHAM CONSONANT SIGN YA..CHAM CONSO... - | 0xEA 0xA8 0xB5..0xB6 #Mn [2] CHAM CONSONANT SIGN LA..CHAM CONSO... - | 0xEA 0xA9 0x83 #Mn CHAM CONSONANT SIGN FINAL NG - | 0xEA 0xA9 0x8C #Mn CHAM CONSONANT SIGN FINAL M - | 0xEA 0xA9 0x8D #Mc CHAM CONSONANT SIGN FINAL H - | 0xEA 0xA9 0xBB #Mc MYANMAR SIGN PAO KAREN TONE - | 0xEA 0xA9 0xBC #Mn MYANMAR SIGN TAI LAING TONE-2 - | 0xEA 0xA9 0xBD #Mc MYANMAR SIGN TAI LAING TONE-5 - | 0xEA 0xAA 0xB0 #Mn TAI VIET MAI KANG - | 0xEA 0xAA 0xB2..0xB4 #Mn [3] TAI VIET VOWEL I..TAI VIET VOWEL U - | 0xEA 0xAA 0xB7..0xB8 #Mn [2] TAI VIET MAI KHIT..TAI VIET VOWEL IA - | 0xEA 0xAA 0xBE..0xBF #Mn [2] TAI VIET VOWEL AM..TAI VIET TONE M... - | 0xEA 0xAB 0x81 #Mn TAI VIET TONE MAI THO - | 0xEA 0xAB 0xAB #Mc MEETEI MAYEK VOWEL SIGN II - | 0xEA 0xAB 0xAC..0xAD #Mn [2] MEETEI MAYEK VOWEL SIGN UU..MEETEI... - | 0xEA 0xAB 0xAE..0xAF #Mc [2] MEETEI MAYEK VOWEL SIGN AU..MEETEI... - | 0xEA 0xAB 0xB5 #Mc MEETEI MAYEK VOWEL SIGN VISARGA - | 0xEA 0xAB 0xB6 #Mn MEETEI MAYEK VIRAMA - | 0xEA 0xAF 0xA3..0xA4 #Mc [2] MEETEI MAYEK VOWEL SIGN ONAP..MEET... - | 0xEA 0xAF 0xA5 #Mn MEETEI MAYEK VOWEL SIGN ANAP - | 0xEA 0xAF 0xA6..0xA7 #Mc [2] MEETEI MAYEK VOWEL SIGN YENAP..MEE... - | 0xEA 0xAF 0xA8 #Mn MEETEI MAYEK VOWEL SIGN UNAP - | 0xEA 0xAF 0xA9..0xAA #Mc [2] MEETEI MAYEK VOWEL SIGN CHEINAP..M... - | 0xEA 0xAF 0xAC #Mc MEETEI MAYEK LUM IYEK - | 0xEA 0xAF 0xAD #Mn MEETEI MAYEK APUN IYEK - | 0xEF 0xAC 0x9E #Mn HEBREW POINT JUDEO-SPANISH VARIKA - | 0xEF 0xB8 0x80..0x8F #Mn [16] VARIATION SELECTOR-1..VARIATION SE... - | 0xEF 0xB8 0xA0..0xAF #Mn [16] COMBINING LIGATURE LEFT HALF..COMB... - | 0xEF 0xBE 0x9E..0x9F #Lm [2] HALFWIDTH KATAKANA VOICED SOUND MA... - | 0xF0 0x90 0x87 0xBD #Mn PHAISTOS DISC SIGN COMBINING OBLIQ... - | 0xF0 0x90 0x8B 0xA0 #Mn COPTIC EPACT THOUSANDS MARK - | 0xF0 0x90 0x8D 0xB6..0xBA #Mn [5] COMBINING OLD PERMIC LETTER AN.... - | 0xF0 0x90 0xA8 0x81..0x83 #Mn [3] KHAROSHTHI VOWEL SIGN I..KHAROS... - | 0xF0 0x90 0xA8 0x85..0x86 #Mn [2] KHAROSHTHI VOWEL SIGN E..KHAROS... - | 0xF0 0x90 0xA8 0x8C..0x8F #Mn [4] KHAROSHTHI VOWEL LENGTH MARK..K... - | 0xF0 0x90 0xA8 0xB8..0xBA #Mn [3] KHAROSHTHI SIGN BAR ABOVE..KHAR... - | 0xF0 0x90 0xA8 0xBF #Mn KHAROSHTHI VIRAMA - | 0xF0 0x90 0xAB 0xA5..0xA6 #Mn [2] MANICHAEAN ABBREVIATION MARK AB... - | 0xF0 0x91 0x80 0x80 #Mc BRAHMI SIGN CANDRABINDU - | 0xF0 0x91 0x80 0x81 #Mn BRAHMI SIGN ANUSVARA - | 0xF0 0x91 0x80 0x82 #Mc BRAHMI SIGN VISARGA - | 0xF0 0x91 0x80 0xB8..0xFF #Mn [15] BRAHMI VOWEL SIGN AA..BRAHMI VI... - | 0xF0 0x91 0x81 0x00..0x86 # - | 0xF0 0x91 0x81 0xBF..0xFF #Mn [3] BRAHMI NUMBER JOINER..KAITHI SI... - | 0xF0 0x91 0x82 0x00..0x81 # - | 0xF0 0x91 0x82 0x82 #Mc KAITHI SIGN VISARGA - | 0xF0 0x91 0x82 0xB0..0xB2 #Mc [3] KAITHI VOWEL SIGN AA..KAITHI VO... - | 0xF0 0x91 0x82 0xB3..0xB6 #Mn [4] KAITHI VOWEL SIGN U..KAITHI VOW... - | 0xF0 0x91 0x82 0xB7..0xB8 #Mc [2] KAITHI VOWEL SIGN O..KAITHI VOW... - | 0xF0 0x91 0x82 0xB9..0xBA #Mn [2] KAITHI SIGN VIRAMA..KAITHI SIGN... - | 0xF0 0x91 0x84 0x80..0x82 #Mn [3] CHAKMA SIGN CANDRABINDU..CHAKMA... - | 0xF0 0x91 0x84 0xA7..0xAB #Mn [5] CHAKMA VOWEL SIGN A..CHAKMA VOW... - | 0xF0 0x91 0x84 0xAC #Mc CHAKMA VOWEL SIGN E - | 0xF0 0x91 0x84 0xAD..0xB4 #Mn [8] CHAKMA VOWEL SIGN AI..CHAKMA MA... - | 0xF0 0x91 0x85 0xB3 #Mn MAHAJANI SIGN NUKTA - | 0xF0 0x91 0x86 0x80..0x81 #Mn [2] SHARADA SIGN CANDRABINDU..SHARA... - | 0xF0 0x91 0x86 0x82 #Mc SHARADA SIGN VISARGA - | 0xF0 0x91 0x86 0xB3..0xB5 #Mc [3] SHARADA VOWEL SIGN AA..SHARADA ... - | 0xF0 0x91 0x86 0xB6..0xBE #Mn [9] SHARADA VOWEL SIGN U..SHARADA V... - | 0xF0 0x91 0x86 0xBF..0xFF #Mc [2] SHARADA VOWEL SIGN AU..SHARADA ... - | 0xF0 0x91 0x87 0x00..0x80 # - | 0xF0 0x91 0x87 0x8A..0x8C #Mn [3] SHARADA SIGN NUKTA..SHARADA EXT... - | 0xF0 0x91 0x88 0xAC..0xAE #Mc [3] KHOJKI VOWEL SIGN AA..KHOJKI VO... - | 0xF0 0x91 0x88 0xAF..0xB1 #Mn [3] KHOJKI VOWEL SIGN U..KHOJKI VOW... - | 0xF0 0x91 0x88 0xB2..0xB3 #Mc [2] KHOJKI VOWEL SIGN O..KHOJKI VOW... - | 0xF0 0x91 0x88 0xB4 #Mn KHOJKI SIGN ANUSVARA - | 0xF0 0x91 0x88 0xB5 #Mc KHOJKI SIGN VIRAMA - | 0xF0 0x91 0x88 0xB6..0xB7 #Mn [2] KHOJKI SIGN NUKTA..KHOJKI SIGN ... - | 0xF0 0x91 0x8B 0x9F #Mn KHUDAWADI SIGN ANUSVARA - | 0xF0 0x91 0x8B 0xA0..0xA2 #Mc [3] KHUDAWADI VOWEL SIGN AA..KHUDAW... - | 0xF0 0x91 0x8B 0xA3..0xAA #Mn [8] KHUDAWADI VOWEL SIGN U..KHUDAWA... - | 0xF0 0x91 0x8C 0x80..0x81 #Mn [2] GRANTHA SIGN COMBINING ANUSVARA... - | 0xF0 0x91 0x8C 0x82..0x83 #Mc [2] GRANTHA SIGN ANUSVARA..GRANTHA ... - | 0xF0 0x91 0x8C 0xBC #Mn GRANTHA SIGN NUKTA - | 0xF0 0x91 0x8C 0xBE..0xBF #Mc [2] GRANTHA VOWEL SIGN AA..GRANTHA ... - | 0xF0 0x91 0x8D 0x80 #Mn GRANTHA VOWEL SIGN II - | 0xF0 0x91 0x8D 0x81..0x84 #Mc [4] GRANTHA VOWEL SIGN U..GRANTHA V... - | 0xF0 0x91 0x8D 0x87..0x88 #Mc [2] GRANTHA VOWEL SIGN EE..GRANTHA ... - | 0xF0 0x91 0x8D 0x8B..0x8D #Mc [3] GRANTHA VOWEL SIGN OO..GRANTHA ... - | 0xF0 0x91 0x8D 0x97 #Mc GRANTHA AU LENGTH MARK - | 0xF0 0x91 0x8D 0xA2..0xA3 #Mc [2] GRANTHA VOWEL SIGN VOCALIC L..G... - | 0xF0 0x91 0x8D 0xA6..0xAC #Mn [7] COMBINING GRANTHA DIGIT ZERO..C... - | 0xF0 0x91 0x8D 0xB0..0xB4 #Mn [5] COMBINING GRANTHA LETTER A..COM... - | 0xF0 0x91 0x92 0xB0..0xB2 #Mc [3] TIRHUTA VOWEL SIGN AA..TIRHUTA ... - | 0xF0 0x91 0x92 0xB3..0xB8 #Mn [6] TIRHUTA VOWEL SIGN U..TIRHUTA V... - | 0xF0 0x91 0x92 0xB9 #Mc TIRHUTA VOWEL SIGN E - | 0xF0 0x91 0x92 0xBA #Mn TIRHUTA VOWEL SIGN SHORT E - | 0xF0 0x91 0x92 0xBB..0xBE #Mc [4] TIRHUTA VOWEL SIGN AI..TIRHUTA ... - | 0xF0 0x91 0x92 0xBF..0xFF #Mn [2] TIRHUTA SIGN CANDRABINDU..TIRHU... - | 0xF0 0x91 0x93 0x00..0x80 # - | 0xF0 0x91 0x93 0x81 #Mc TIRHUTA SIGN VISARGA - | 0xF0 0x91 0x93 0x82..0x83 #Mn [2] TIRHUTA SIGN VIRAMA..TIRHUTA SI... - | 0xF0 0x91 0x96 0xAF..0xB1 #Mc [3] SIDDHAM VOWEL SIGN AA..SIDDHAM ... - | 0xF0 0x91 0x96 0xB2..0xB5 #Mn [4] SIDDHAM VOWEL SIGN U..SIDDHAM V... - | 0xF0 0x91 0x96 0xB8..0xBB #Mc [4] SIDDHAM VOWEL SIGN E..SIDDHAM V... - | 0xF0 0x91 0x96 0xBC..0xBD #Mn [2] SIDDHAM SIGN CANDRABINDU..SIDDH... - | 0xF0 0x91 0x96 0xBE #Mc SIDDHAM SIGN VISARGA - | 0xF0 0x91 0x96 0xBF..0xFF #Mn [2] SIDDHAM SIGN VIRAMA..SIDDHAM SI... - | 0xF0 0x91 0x97 0x00..0x80 # - | 0xF0 0x91 0x97 0x9C..0x9D #Mn [2] SIDDHAM VOWEL SIGN ALTERNATE U.... - | 0xF0 0x91 0x98 0xB0..0xB2 #Mc [3] MODI VOWEL SIGN AA..MODI VOWEL ... - | 0xF0 0x91 0x98 0xB3..0xBA #Mn [8] MODI VOWEL SIGN U..MODI VOWEL S... - | 0xF0 0x91 0x98 0xBB..0xBC #Mc [2] MODI VOWEL SIGN O..MODI VOWEL S... - | 0xF0 0x91 0x98 0xBD #Mn MODI SIGN ANUSVARA - | 0xF0 0x91 0x98 0xBE #Mc MODI SIGN VISARGA - | 0xF0 0x91 0x98 0xBF..0xFF #Mn [2] MODI SIGN VIRAMA..MODI SIGN ARD... - | 0xF0 0x91 0x99 0x00..0x80 # - | 0xF0 0x91 0x9A 0xAB #Mn TAKRI SIGN ANUSVARA - | 0xF0 0x91 0x9A 0xAC #Mc TAKRI SIGN VISARGA - | 0xF0 0x91 0x9A 0xAD #Mn TAKRI VOWEL SIGN AA - | 0xF0 0x91 0x9A 0xAE..0xAF #Mc [2] TAKRI VOWEL SIGN I..TAKRI VOWEL... - | 0xF0 0x91 0x9A 0xB0..0xB5 #Mn [6] TAKRI VOWEL SIGN U..TAKRI VOWEL... - | 0xF0 0x91 0x9A 0xB6 #Mc TAKRI SIGN VIRAMA - | 0xF0 0x91 0x9A 0xB7 #Mn TAKRI SIGN NUKTA - | 0xF0 0x91 0x9C 0x9D..0x9F #Mn [3] AHOM CONSONANT SIGN MEDIAL LA..... - | 0xF0 0x91 0x9C 0xA0..0xA1 #Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL S... - | 0xF0 0x91 0x9C 0xA2..0xA5 #Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL S... - | 0xF0 0x91 0x9C 0xA6 #Mc AHOM VOWEL SIGN E - | 0xF0 0x91 0x9C 0xA7..0xAB #Mn [5] AHOM VOWEL SIGN AW..AHOM SIGN K... - | 0xF0 0x96 0xAB 0xB0..0xB4 #Mn [5] BASSA VAH COMBINING HIGH TONE..... - | 0xF0 0x96 0xAC 0xB0..0xB6 #Mn [7] PAHAWH HMONG MARK CIM TUB..PAHA... - | 0xF0 0x96 0xBD 0x91..0xBE #Mc [46] MIAO SIGN ASPIRATION..MIAO VOWE... - | 0xF0 0x96 0xBE 0x8F..0x92 #Mn [4] MIAO TONE RIGHT..MIAO TONE BELOW - | 0xF0 0x9B 0xB2 0x9D..0x9E #Mn [2] DUPLOYAN THICK LETTER SELECTOR.... - | 0xF0 0x9D 0x85 0xA5..0xA6 #Mc [2] MUSICAL SYMBOL COMBINING STEM..... - | 0xF0 0x9D 0x85 0xA7..0xA9 #Mn [3] MUSICAL SYMBOL COMBINING TREMOL... - | 0xF0 0x9D 0x85 0xAD..0xB2 #Mc [6] MUSICAL SYMBOL COMBINING AUGMEN... - | 0xF0 0x9D 0x85 0xBB..0xFF #Mn [8] MUSICAL SYMBOL COMBINING ACCENT... - | 0xF0 0x9D 0x86 0x00..0x82 # - | 0xF0 0x9D 0x86 0x85..0x8B #Mn [7] MUSICAL SYMBOL COMBINING DOIT..... - | 0xF0 0x9D 0x86 0xAA..0xAD #Mn [4] MUSICAL SYMBOL COMBINING DOWN B... - | 0xF0 0x9D 0x89 0x82..0x84 #Mn [3] COMBINING GREEK MUSICAL TRISEME... - | 0xF0 0x9D 0xA8 0x80..0xB6 #Mn [55] SIGNWRITING HEAD RIM..SIGNWRITI... - | 0xF0 0x9D 0xA8 0xBB..0xFF #Mn [50] SIGNWRITING MOUTH CLOSED NEUTRA... - | 0xF0 0x9D 0xA9 0x00..0xAC # - | 0xF0 0x9D 0xA9 0xB5 #Mn SIGNWRITING UPPER BODY TILTING FRO... - | 0xF0 0x9D 0xAA 0x84 #Mn SIGNWRITING LOCATION HEAD NECK - | 0xF0 0x9D 0xAA 0x9B..0x9F #Mn [5] SIGNWRITING FILL MODIFIER-2..SI... - | 0xF0 0x9D 0xAA 0xA1..0xAF #Mn [15] SIGNWRITING ROTATION MODIFIER-2... - | 0xF0 0x9E 0xA3 0x90..0x96 #Mn [7] MENDE KIKAKUI COMBINING NUMBER ... - | 0xF3 0xA0 0x84 0x80..0xFF #Mn [240] VARIATION SELECTOR-17..VA... - | 0xF3 0xA0 0x85..0x86 0x00..0xFF # - | 0xF3 0xA0 0x87 0x00..0xAF # - ; - - Format = - 0xC2 0xAD #Cf SOFT HYPHEN - | 0xD8 0x80..0x85 #Cf [6] ARABIC NUMBER SIGN..ARABIC NUMBER ... - | 0xD8 0x9C #Cf ARABIC LETTER MARK - | 0xDB 0x9D #Cf ARABIC END OF AYAH - | 0xDC 0x8F #Cf SYRIAC ABBREVIATION MARK - | 0xE1 0xA0 0x8E #Cf MONGOLIAN VOWEL SEPARATOR - | 0xE2 0x80 0x8E..0x8F #Cf [2] LEFT-TO-RIGHT MARK..RIGHT-TO-LEFT ... - | 0xE2 0x80 0xAA..0xAE #Cf [5] LEFT-TO-RIGHT EMBEDDING..RIGHT-TO-... - | 0xE2 0x81 0xA0..0xA4 #Cf [5] WORD JOINER..INVISIBLE PLUS - | 0xE2 0x81 0xA6..0xAF #Cf [10] LEFT-TO-RIGHT ISOLATE..NOMINAL DIG... - | 0xEF 0xBB 0xBF #Cf ZERO WIDTH NO-BREAK SPACE - | 0xEF 0xBF 0xB9..0xBB #Cf [3] INTERLINEAR ANNOTATION ANCHOR..INT... - | 0xF0 0x91 0x82 0xBD #Cf KAITHI NUMBER SIGN - | 0xF0 0x9B 0xB2 0xA0..0xA3 #Cf [4] SHORTHAND FORMAT LETTER OVERLAP... - | 0xF0 0x9D 0x85 0xB3..0xBA #Cf [8] MUSICAL SYMBOL BEGIN BEAM..MUSI... - | 0xF3 0xA0 0x80 0x81 #Cf LANGUAGE TAG - | 0xF3 0xA0 0x80 0xA0..0xFF #Cf [96] TAG SPACE..CANCEL TAG - | 0xF3 0xA0 0x81 0x00..0xBF # - ; - - Katakana = - 0xE3 0x80 0xB1..0xB5 #Lm [5] VERTICAL KANA REPEAT MARK..VERTICA... - | 0xE3 0x82 0x9B..0x9C #Sk [2] KATAKANA-HIRAGANA VOICED SOUND MAR... - | 0xE3 0x82 0xA0 #Pd KATAKANA-HIRAGANA DOUBLE HYPHEN - | 0xE3 0x82 0xA1..0xFF #Lo [90] KATAKANA LETTER SMALL A..KATAKANA ... - | 0xE3 0x83 0x00..0xBA # - | 0xE3 0x83 0xBC..0xBE #Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND ... - | 0xE3 0x83 0xBF #Lo KATAKANA DIGRAPH KOTO - | 0xE3 0x87 0xB0..0xBF #Lo [16] KATAKANA LETTER SMALL KU..KATAKANA... - | 0xE3 0x8B 0x90..0xBE #So [47] CIRCLED KATAKANA A..CIRCLED KATAKA... - | 0xE3 0x8C 0x80..0xFF #So [88] SQUARE APAATO..SQUARE WATTO - | 0xE3 0x8D 0x00..0x97 # - | 0xEF 0xBD 0xA6..0xAF #Lo [10] HALFWIDTH KATAKANA LETTER WO..HALF... - | 0xEF 0xBD 0xB0 #Lm HALFWIDTH KATAKANA-HIRAGANA PROLON... - | 0xEF 0xBD 0xB1..0xFF #Lo [45] HALFWIDTH KATAKANA LETTER A..HALFW... - | 0xEF 0xBE 0x00..0x9D # - | 0xF0 0x9B 0x80 0x80 #Lo KATAKANA LETTER ARCHAIC E - ; - - ALetter = - 0x41..0x5A #L& [26] LATIN CAPITAL LETTER A..LATIN CAPI... - | 0x61..0x7A #L& [26] LATIN SMALL LETTER A..LATIN SMALL ... - | 0xC2 0xAA #Lo FEMININE ORDINAL INDICATOR - | 0xC2 0xB5 #L& MICRO SIGN - | 0xC2 0xBA #Lo MASCULINE ORDINAL INDICATOR - | 0xC3 0x80..0x96 #L& [23] LATIN CAPITAL LETTER A WITH GRAVE.... - | 0xC3 0x98..0xB6 #L& [31] LATIN CAPITAL LETTER O WITH STROKE... - | 0xC3 0xB8..0xFF #L& [195] LATIN SMALL LETTER O WITH STROKE..... - | 0xC4..0xC5 0x00..0xFF # - | 0xC6 0x00..0xBA # - | 0xC6 0xBB #Lo LATIN LETTER TWO WITH STROKE - | 0xC6 0xBC..0xBF #L& [4] LATIN CAPITAL LETTER TONE FIVE..LA... - | 0xC7 0x80..0x83 #Lo [4] LATIN LETTER DENTAL CLICK..LATIN L... - | 0xC7 0x84..0xFF #L& [208] LATIN CAPITAL LETTER DZ WITH CARON... - | 0xC8..0xC9 0x00..0xFF # - | 0xCA 0x00..0x93 # - | 0xCA 0x94 #Lo LATIN LETTER GLOTTAL STOP - | 0xCA 0x95..0xAF #L& [27] LATIN LETTER PHARYNGEAL VOICED FRI... - | 0xCA 0xB0..0xFF #Lm [18] MODIFIER LETTER SMALL H..MODIFIER ... - | 0xCB 0x00..0x81 # - | 0xCB 0x86..0x91 #Lm [12] MODIFIER LETTER CIRCUMFLEX ACCENT.... - | 0xCB 0xA0..0xA4 #Lm [5] MODIFIER LETTER SMALL GAMMA..MODIF... - | 0xCB 0xAC #Lm MODIFIER LETTER VOICING - | 0xCB 0xAE #Lm MODIFIER LETTER DOUBLE APOSTROPHE - | 0xCD 0xB0..0xB3 #L& [4] GREEK CAPITAL LETTER HETA..GREEK S... - | 0xCD 0xB4 #Lm GREEK NUMERAL SIGN - | 0xCD 0xB6..0xB7 #L& [2] GREEK CAPITAL LETTER PAMPHYLIAN DI... - | 0xCD 0xBA #Lm GREEK YPOGEGRAMMENI - | 0xCD 0xBB..0xBD #L& [3] GREEK SMALL REVERSED LUNATE SIGMA ... - | 0xCD 0xBF #L& GREEK CAPITAL LETTER YOT - | 0xCE 0x86 #L& GREEK CAPITAL LETTER ALPHA WITH TONOS - | 0xCE 0x88..0x8A #L& [3] GREEK CAPITAL LETTER EPSILON WITH ... - | 0xCE 0x8C #L& GREEK CAPITAL LETTER OMICRON WITH ... - | 0xCE 0x8E..0xA1 #L& [20] GREEK CAPITAL LETTER UPSILON WITH ... - | 0xCE 0xA3..0xFF #L& [83] GREEK CAPITAL LETTER SIGMA..GREEK ... - | 0xCF 0x00..0xB5 # - | 0xCF 0xB7..0xFF #L& [139] GREEK CAPITAL LETTER SHO..CYRILLIC... - | 0xD0..0xD1 0x00..0xFF # - | 0xD2 0x00..0x81 # - | 0xD2 0x8A..0xFF #L& [166] CYRILLIC CAPITAL LETTER SHORT I WI... - | 0xD3..0xD3 0x00..0xFF # - | 0xD4 0x00..0xAF # - | 0xD4 0xB1..0xFF #L& [38] ARMENIAN CAPITAL LETTER AYB..ARMEN... - | 0xD5 0x00..0x96 # - | 0xD5 0x99 #Lm ARMENIAN MODIFIER LETTER LEFT HALF... - | 0xD5 0xA1..0xFF #L& [39] ARMENIAN SMALL LETTER AYB..ARMENIA... - | 0xD6 0x00..0x87 # - | 0xD7 0xB3 #Po HEBREW PUNCTUATION GERESH - | 0xD8 0xA0..0xBF #Lo [32] ARABIC LETTER KASHMIRI YEH..ARABIC... - | 0xD9 0x80 #Lm ARABIC TATWEEL - | 0xD9 0x81..0x8A #Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH - | 0xD9 0xAE..0xAF #Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC ... - | 0xD9 0xB1..0xFF #Lo [99] ARABIC LETTER ALEF WASLA..ARABIC L... - | 0xDA..0xDA 0x00..0xFF # - | 0xDB 0x00..0x93 # - | 0xDB 0x95 #Lo ARABIC LETTER AE - | 0xDB 0xA5..0xA6 #Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH - | 0xDB 0xAE..0xAF #Lo [2] ARABIC LETTER DAL WITH INVERTED V.... - | 0xDB 0xBA..0xBC #Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW... - | 0xDB 0xBF #Lo ARABIC LETTER HEH WITH INVERTED V - | 0xDC 0x90 #Lo SYRIAC LETTER ALAPH - | 0xDC 0x92..0xAF #Lo [30] SYRIAC LETTER BETH..SYRIAC LETTER ... - | 0xDD 0x8D..0xFF #Lo [89] SYRIAC LETTER SOGDIAN ZHAIN..THAAN... - | 0xDE 0x00..0xA5 # - | 0xDE 0xB1 #Lo THAANA LETTER NAA - | 0xDF 0x8A..0xAA #Lo [33] NKO LETTER A..NKO LETTER JONA RA - | 0xDF 0xB4..0xB5 #Lm [2] NKO HIGH TONE APOSTROPHE..NKO LOW ... - | 0xDF 0xBA #Lm NKO LAJANYALAN - | 0xE0 0xA0 0x80..0x95 #Lo [22] SAMARITAN LETTER ALAF..SAMARITAN L... - | 0xE0 0xA0 0x9A #Lm SAMARITAN MODIFIER LETTER EPENTHET... - | 0xE0 0xA0 0xA4 #Lm SAMARITAN MODIFIER LETTER SHORT A - | 0xE0 0xA0 0xA8 #Lm SAMARITAN MODIFIER LETTER I - | 0xE0 0xA1 0x80..0x98 #Lo [25] MANDAIC LETTER HALQA..MANDAIC LETT... - | 0xE0 0xA2 0xA0..0xB4 #Lo [21] ARABIC LETTER BEH WITH SMALL V BEL... - | 0xE0 0xA4 0x84..0xB9 #Lo [54] DEVANAGARI LETTER SHORT A..DEVANAG... - | 0xE0 0xA4 0xBD #Lo DEVANAGARI SIGN AVAGRAHA - | 0xE0 0xA5 0x90 #Lo DEVANAGARI OM - | 0xE0 0xA5 0x98..0xA1 #Lo [10] DEVANAGARI LETTER QA..DEVANAGARI L... - | 0xE0 0xA5 0xB1 #Lm DEVANAGARI SIGN HIGH SPACING DOT - | 0xE0 0xA5 0xB2..0xFF #Lo [15] DEVANAGARI LETTER CANDRA A..BENGAL... - | 0xE0 0xA6 0x00..0x80 # - | 0xE0 0xA6 0x85..0x8C #Lo [8] BENGALI LETTER A..BENGALI LETTER V... - | 0xE0 0xA6 0x8F..0x90 #Lo [2] BENGALI LETTER E..BENGALI LETTER AI - | 0xE0 0xA6 0x93..0xA8 #Lo [22] BENGALI LETTER O..BENGALI LETTER NA - | 0xE0 0xA6 0xAA..0xB0 #Lo [7] BENGALI LETTER PA..BENGALI LETTER RA - | 0xE0 0xA6 0xB2 #Lo BENGALI LETTER LA - | 0xE0 0xA6 0xB6..0xB9 #Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA - | 0xE0 0xA6 0xBD #Lo BENGALI SIGN AVAGRAHA - | 0xE0 0xA7 0x8E #Lo BENGALI LETTER KHANDA TA - | 0xE0 0xA7 0x9C..0x9D #Lo [2] BENGALI LETTER RRA..BENGALI LETTER... - | 0xE0 0xA7 0x9F..0xA1 #Lo [3] BENGALI LETTER YYA..BENGALI LETTER... - | 0xE0 0xA7 0xB0..0xB1 #Lo [2] BENGALI LETTER RA WITH MIDDLE DIAG... - | 0xE0 0xA8 0x85..0x8A #Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU - | 0xE0 0xA8 0x8F..0x90 #Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTE... - | 0xE0 0xA8 0x93..0xA8 #Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTE... - | 0xE0 0xA8 0xAA..0xB0 #Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTE... - | 0xE0 0xA8 0xB2..0xB3 #Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTE... - | 0xE0 0xA8 0xB5..0xB6 #Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTE... - | 0xE0 0xA8 0xB8..0xB9 #Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTE... - | 0xE0 0xA9 0x99..0x9C #Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LET... - | 0xE0 0xA9 0x9E #Lo GURMUKHI LETTER FA - | 0xE0 0xA9 0xB2..0xB4 #Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR - | 0xE0 0xAA 0x85..0x8D #Lo [9] GUJARATI LETTER A..GUJARATI VOWEL ... - | 0xE0 0xAA 0x8F..0x91 #Lo [3] GUJARATI LETTER E..GUJARATI VOWEL ... - | 0xE0 0xAA 0x93..0xA8 #Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA - | 0xE0 0xAA 0xAA..0xB0 #Lo [7] GUJARATI LETTER PA..GUJARATI LETTE... - | 0xE0 0xAA 0xB2..0xB3 #Lo [2] GUJARATI LETTER LA..GUJARATI LETTE... - | 0xE0 0xAA 0xB5..0xB9 #Lo [5] GUJARATI LETTER VA..GUJARATI LETTE... - | 0xE0 0xAA 0xBD #Lo GUJARATI SIGN AVAGRAHA - | 0xE0 0xAB 0x90 #Lo GUJARATI OM - | 0xE0 0xAB 0xA0..0xA1 #Lo [2] GUJARATI LETTER VOCALIC RR..GUJARA... - | 0xE0 0xAB 0xB9 #Lo GUJARATI LETTER ZHA - | 0xE0 0xAC 0x85..0x8C #Lo [8] ORIYA LETTER A..ORIYA LETTER VOCAL... - | 0xE0 0xAC 0x8F..0x90 #Lo [2] ORIYA LETTER E..ORIYA LETTER AI - | 0xE0 0xAC 0x93..0xA8 #Lo [22] ORIYA LETTER O..ORIYA LETTER NA - | 0xE0 0xAC 0xAA..0xB0 #Lo [7] ORIYA LETTER PA..ORIYA LETTER RA - | 0xE0 0xAC 0xB2..0xB3 #Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA - | 0xE0 0xAC 0xB5..0xB9 #Lo [5] ORIYA LETTER VA..ORIYA LETTER HA - | 0xE0 0xAC 0xBD #Lo ORIYA SIGN AVAGRAHA - | 0xE0 0xAD 0x9C..0x9D #Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA - | 0xE0 0xAD 0x9F..0xA1 #Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOC... - | 0xE0 0xAD 0xB1 #Lo ORIYA LETTER WA - | 0xE0 0xAE 0x83 #Lo TAMIL SIGN VISARGA - | 0xE0 0xAE 0x85..0x8A #Lo [6] TAMIL LETTER A..TAMIL LETTER UU - | 0xE0 0xAE 0x8E..0x90 #Lo [3] TAMIL LETTER E..TAMIL LETTER AI - | 0xE0 0xAE 0x92..0x95 #Lo [4] TAMIL LETTER O..TAMIL LETTER KA - | 0xE0 0xAE 0x99..0x9A #Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA - | 0xE0 0xAE 0x9C #Lo TAMIL LETTER JA - | 0xE0 0xAE 0x9E..0x9F #Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA - | 0xE0 0xAE 0xA3..0xA4 #Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA - | 0xE0 0xAE 0xA8..0xAA #Lo [3] TAMIL LETTER NA..TAMIL LETTER PA - | 0xE0 0xAE 0xAE..0xB9 #Lo [12] TAMIL LETTER MA..TAMIL LETTER HA - | 0xE0 0xAF 0x90 #Lo TAMIL OM - | 0xE0 0xB0 0x85..0x8C #Lo [8] TELUGU LETTER A..TELUGU LETTER VOC... - | 0xE0 0xB0 0x8E..0x90 #Lo [3] TELUGU LETTER E..TELUGU LETTER AI - | 0xE0 0xB0 0x92..0xA8 #Lo [23] TELUGU LETTER O..TELUGU LETTER NA - | 0xE0 0xB0 0xAA..0xB9 #Lo [16] TELUGU LETTER PA..TELUGU LETTER HA - | 0xE0 0xB0 0xBD #Lo TELUGU SIGN AVAGRAHA - | 0xE0 0xB1 0x98..0x9A #Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA - | 0xE0 0xB1 0xA0..0xA1 #Lo [2] TELUGU LETTER VOCALIC RR..TELUGU L... - | 0xE0 0xB2 0x85..0x8C #Lo [8] KANNADA LETTER A..KANNADA LETTER V... - | 0xE0 0xB2 0x8E..0x90 #Lo [3] KANNADA LETTER E..KANNADA LETTER AI - | 0xE0 0xB2 0x92..0xA8 #Lo [23] KANNADA LETTER O..KANNADA LETTER NA - | 0xE0 0xB2 0xAA..0xB3 #Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA - | 0xE0 0xB2 0xB5..0xB9 #Lo [5] KANNADA LETTER VA..KANNADA LETTER HA - | 0xE0 0xB2 0xBD #Lo KANNADA SIGN AVAGRAHA - | 0xE0 0xB3 0x9E #Lo KANNADA LETTER FA - | 0xE0 0xB3 0xA0..0xA1 #Lo [2] KANNADA LETTER VOCALIC RR..KANNADA... - | 0xE0 0xB3 0xB1..0xB2 #Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA ... - | 0xE0 0xB4 0x85..0x8C #Lo [8] MALAYALAM LETTER A..MALAYALAM LETT... - | 0xE0 0xB4 0x8E..0x90 #Lo [3] MALAYALAM LETTER E..MALAYALAM LETT... - | 0xE0 0xB4 0x92..0xBA #Lo [41] MALAYALAM LETTER O..MALAYALAM LETT... - | 0xE0 0xB4 0xBD #Lo MALAYALAM SIGN AVAGRAHA - | 0xE0 0xB5 0x8E #Lo MALAYALAM LETTER DOT REPH - | 0xE0 0xB5 0x9F..0xA1 #Lo [3] MALAYALAM LETTER ARCHAIC II..MALAY... - | 0xE0 0xB5 0xBA..0xBF #Lo [6] MALAYALAM LETTER CHILLU NN..MALAYA... - | 0xE0 0xB6 0x85..0x96 #Lo [18] SINHALA LETTER AYANNA..SINHALA LET... - | 0xE0 0xB6 0x9A..0xB1 #Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA.... - | 0xE0 0xB6 0xB3..0xBB #Lo [9] SINHALA LETTER SANYAKA DAYANNA..SI... - | 0xE0 0xB6 0xBD #Lo SINHALA LETTER DANTAJA LAYANNA - | 0xE0 0xB7 0x80..0x86 #Lo [7] SINHALA LETTER VAYANNA..SINHALA LE... - | 0xE0 0xBC 0x80 #Lo TIBETAN SYLLABLE OM - | 0xE0 0xBD 0x80..0x87 #Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA - | 0xE0 0xBD 0x89..0xAC #Lo [36] TIBETAN LETTER NYA..TIBETAN LETTER... - | 0xE0 0xBE 0x88..0x8C #Lo [5] TIBETAN SIGN LCE TSA CAN..TIBETAN ... - | 0xE1 0x82 0xA0..0xFF #L& [38] GEORGIAN CAPITAL LETTER AN..GEORGI... - | 0xE1 0x83 0x00..0x85 # - | 0xE1 0x83 0x87 #L& GEORGIAN CAPITAL LETTER YN - | 0xE1 0x83 0x8D #L& GEORGIAN CAPITAL LETTER AEN - | 0xE1 0x83 0x90..0xBA #Lo [43] GEORGIAN LETTER AN..GEORGIAN LETTE... - | 0xE1 0x83 0xBC #Lm MODIFIER LETTER GEORGIAN NAR - | 0xE1 0x83 0xBD..0xFF #Lo [332] GEORGIAN LETTER AEN..ETHIOPIC ... - | 0xE1 0x84..0x88 0x00..0xFF # - | 0xE1 0x89 0x00..0x88 # - | 0xE1 0x89 0x8A..0x8D #Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SY... - | 0xE1 0x89 0x90..0x96 #Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SY... - | 0xE1 0x89 0x98 #Lo ETHIOPIC SYLLABLE QHWA - | 0xE1 0x89 0x9A..0x9D #Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC S... - | 0xE1 0x89 0xA0..0xFF #Lo [41] ETHIOPIC SYLLABLE BA..ETHIOPIC SYL... - | 0xE1 0x8A 0x00..0x88 # - | 0xE1 0x8A 0x8A..0x8D #Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SY... - | 0xE1 0x8A 0x90..0xB0 #Lo [33] ETHIOPIC SYLLABLE NA..ETHIOPIC SYL... - | 0xE1 0x8A 0xB2..0xB5 #Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SY... - | 0xE1 0x8A 0xB8..0xBE #Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SY... - | 0xE1 0x8B 0x80 #Lo ETHIOPIC SYLLABLE KXWA - | 0xE1 0x8B 0x82..0x85 #Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC S... - | 0xE1 0x8B 0x88..0x96 #Lo [15] ETHIOPIC SYLLABLE WA..ETHIOPIC SYL... - | 0xE1 0x8B 0x98..0xFF #Lo [57] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYL... - | 0xE1 0x8C 0x00..0x90 # - | 0xE1 0x8C 0x92..0x95 #Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SY... - | 0xE1 0x8C 0x98..0xFF #Lo [67] ETHIOPIC SYLLABLE GGA..ETHIOPIC SY... - | 0xE1 0x8D 0x00..0x9A # - | 0xE1 0x8E 0x80..0x8F #Lo [16] ETHIOPIC SYLLABLE SEBATBEIT MWA..E... - | 0xE1 0x8E 0xA0..0xFF #L& [86] CHEROKEE LETTER A..CHEROKEE LETTER MV - | 0xE1 0x8F 0x00..0xB5 # - | 0xE1 0x8F 0xB8..0xBD #L& [6] CHEROKEE SMALL LETTER YE..CHEROKEE... - | 0xE1 0x90 0x81..0xFF #Lo [620] CANADIAN SYLLABICS E..CANADIAN... - | 0xE1 0x91..0x98 0x00..0xFF # - | 0xE1 0x99 0x00..0xAC # - | 0xE1 0x99 0xAF..0xBF #Lo [17] CANADIAN SYLLABICS QAI..CANADIAN S... - | 0xE1 0x9A 0x81..0x9A #Lo [26] OGHAM LETTER BEITH..OGHAM LETTER P... - | 0xE1 0x9A 0xA0..0xFF #Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC... - | 0xE1 0x9B 0x00..0xAA # - | 0xE1 0x9B 0xAE..0xB0 #Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHO... - | 0xE1 0x9B 0xB1..0xB8 #Lo [8] RUNIC LETTER K..RUNIC LETTER FRANK... - | 0xE1 0x9C 0x80..0x8C #Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA - | 0xE1 0x9C 0x8E..0x91 #Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA - | 0xE1 0x9C 0xA0..0xB1 #Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA - | 0xE1 0x9D 0x80..0x91 #Lo [18] BUHID LETTER A..BUHID LETTER HA - | 0xE1 0x9D 0xA0..0xAC #Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA - | 0xE1 0x9D 0xAE..0xB0 #Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTE... - | 0xE1 0xA0 0xA0..0xFF #Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETT... - | 0xE1 0xA1 0x00..0x82 # - | 0xE1 0xA1 0x83 #Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN - | 0xE1 0xA1 0x84..0xB7 #Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN... - | 0xE1 0xA2 0x80..0xA8 #Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA... - | 0xE1 0xA2 0xAA #Lo MONGOLIAN LETTER MANCHU ALI GALI LHA - | 0xE1 0xA2 0xB0..0xFF #Lo [70] CANADIAN SYLLABICS OY..CANADIAN SY... - | 0xE1 0xA3 0x00..0xB5 # - | 0xE1 0xA4 0x80..0x9E #Lo [31] LIMBU VOWEL-CARRIER LETTER..LIMBU ... - | 0xE1 0xA8 0x80..0x96 #Lo [23] BUGINESE LETTER KA..BUGINESE LETTE... - | 0xE1 0xAC 0x85..0xB3 #Lo [47] BALINESE LETTER AKARA..BALINESE LE... - | 0xE1 0xAD 0x85..0x8B #Lo [7] BALINESE LETTER KAF SASAK..BALINES... - | 0xE1 0xAE 0x83..0xA0 #Lo [30] SUNDANESE LETTER A..SUNDANESE LETT... - | 0xE1 0xAE 0xAE..0xAF #Lo [2] SUNDANESE LETTER KHA..SUNDANESE LE... - | 0xE1 0xAE 0xBA..0xFF #Lo [44] SUNDANESE AVAGRAHA..BATAK LETTER U - | 0xE1 0xAF 0x00..0xA5 # - | 0xE1 0xB0 0x80..0xA3 #Lo [36] LEPCHA LETTER KA..LEPCHA LETTER A - | 0xE1 0xB1 0x8D..0x8F #Lo [3] LEPCHA LETTER TTA..LEPCHA LETTER DDA - | 0xE1 0xB1 0x9A..0xB7 #Lo [30] OL CHIKI LETTER LA..OL CHIKI LETTE... - | 0xE1 0xB1 0xB8..0xBD #Lm [6] OL CHIKI MU TTUDDAG..OL CHIKI AHAD - | 0xE1 0xB3 0xA9..0xAC #Lo [4] VEDIC SIGN ANUSVARA ANTARGOMUKHA..... - | 0xE1 0xB3 0xAE..0xB1 #Lo [4] VEDIC SIGN HEXIFORM LONG ANUSVARA.... - | 0xE1 0xB3 0xB5..0xB6 #Lo [2] VEDIC SIGN JIHVAMULIYA..VEDIC SIGN... - | 0xE1 0xB4 0x80..0xAB #L& [44] LATIN LETTER SMALL CAPITAL A..CYRI... - | 0xE1 0xB4 0xAC..0xFF #Lm [63] MODIFIER LETTER CAPITAL A..GREEK S... - | 0xE1 0xB5 0x00..0xAA # - | 0xE1 0xB5 0xAB..0xB7 #L& [13] LATIN SMALL LETTER UE..LATIN SMALL... - | 0xE1 0xB5 0xB8 #Lm MODIFIER LETTER CYRILLIC EN - | 0xE1 0xB5 0xB9..0xFF #L& [34] LATIN SMALL LETTER INSULAR G..LATI... - | 0xE1 0xB6 0x00..0x9A # - | 0xE1 0xB6 0x9B..0xBF #Lm [37] MODIFIER LETTER SMALL TURNED ALPHA... - | 0xE1 0xB8 0x80..0xFF #L& [278] LATIN CAPITAL LETTER A WITH RI... - | 0xE1 0xB9..0xBB 0x00..0xFF # - | 0xE1 0xBC 0x00..0x95 # - | 0xE1 0xBC 0x98..0x9D #L& [6] GREEK CAPITAL LETTER EPSILON WITH ... - | 0xE1 0xBC 0xA0..0xFF #L& [38] GREEK SMALL LETTER ETA WITH PSILI.... - | 0xE1 0xBD 0x00..0x85 # - | 0xE1 0xBD 0x88..0x8D #L& [6] GREEK CAPITAL LETTER OMICRON WITH ... - | 0xE1 0xBD 0x90..0x97 #L& [8] GREEK SMALL LETTER UPSILON WITH PS... - | 0xE1 0xBD 0x99 #L& GREEK CAPITAL LETTER UPSILON WITH ... - | 0xE1 0xBD 0x9B #L& GREEK CAPITAL LETTER UPSILON WITH ... - | 0xE1 0xBD 0x9D #L& GREEK CAPITAL LETTER UPSILON WITH ... - | 0xE1 0xBD 0x9F..0xBD #L& [31] GREEK CAPITAL LETTER UPSILON WITH ... - | 0xE1 0xBE 0x80..0xB4 #L& [53] GREEK SMALL LETTER ALPHA WITH PSIL... - | 0xE1 0xBE 0xB6..0xBC #L& [7] GREEK SMALL LETTER ALPHA WITH PERI... - | 0xE1 0xBE 0xBE #L& GREEK PROSGEGRAMMENI - | 0xE1 0xBF 0x82..0x84 #L& [3] GREEK SMALL LETTER ETA WITH VARIA ... - | 0xE1 0xBF 0x86..0x8C #L& [7] GREEK SMALL LETTER ETA WITH PERISP... - | 0xE1 0xBF 0x90..0x93 #L& [4] GREEK SMALL LETTER IOTA WITH VRACH... - | 0xE1 0xBF 0x96..0x9B #L& [6] GREEK SMALL LETTER IOTA WITH PERIS... - | 0xE1 0xBF 0xA0..0xAC #L& [13] GREEK SMALL LETTER UPSILON WITH VR... - | 0xE1 0xBF 0xB2..0xB4 #L& [3] GREEK SMALL LETTER OMEGA WITH VARI... - | 0xE1 0xBF 0xB6..0xBC #L& [7] GREEK SMALL LETTER OMEGA WITH PERI... - | 0xE2 0x81 0xB1 #Lm SUPERSCRIPT LATIN SMALL LETTER I - | 0xE2 0x81 0xBF #Lm SUPERSCRIPT LATIN SMALL LETTER N - | 0xE2 0x82 0x90..0x9C #Lm [13] LATIN SUBSCRIPT SMALL LETTER A..LA... - | 0xE2 0x84 0x82 #L& DOUBLE-STRUCK CAPITAL C - | 0xE2 0x84 0x87 #L& EULER CONSTANT - | 0xE2 0x84 0x8A..0x93 #L& [10] SCRIPT SMALL G..SCRIPT SMALL L - | 0xE2 0x84 0x95 #L& DOUBLE-STRUCK CAPITAL N - | 0xE2 0x84 0x99..0x9D #L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-ST... - | 0xE2 0x84 0xA4 #L& DOUBLE-STRUCK CAPITAL Z - | 0xE2 0x84 0xA6 #L& OHM SIGN - | 0xE2 0x84 0xA8 #L& BLACK-LETTER CAPITAL Z - | 0xE2 0x84 0xAA..0xAD #L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C - | 0xE2 0x84 0xAF..0xB4 #L& [6] SCRIPT SMALL E..SCRIPT SMALL O - | 0xE2 0x84 0xB5..0xB8 #Lo [4] ALEF SYMBOL..DALET SYMBOL - | 0xE2 0x84 0xB9 #L& INFORMATION SOURCE - | 0xE2 0x84 0xBC..0xBF #L& [4] DOUBLE-STRUCK SMALL PI..DOUBLE-STR... - | 0xE2 0x85 0x85..0x89 #L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DO... - | 0xE2 0x85 0x8E #L& TURNED SMALL F - | 0xE2 0x85 0xA0..0xFF #Nl [35] ROMAN NUMERAL ONE..ROMAN NUMERAL T... - | 0xE2 0x86 0x00..0x82 # - | 0xE2 0x86 0x83..0x84 #L& [2] ROMAN NUMERAL REVERSED ONE HUNDRED... - | 0xE2 0x86 0x85..0x88 #Nl [4] ROMAN NUMERAL SIX LATE FORM..ROMAN... - | 0xE2 0x92 0xB6..0xFF #So [52] CIRCLED LATIN CAPITAL LETTER A..CI... - | 0xE2 0x93 0x00..0xA9 # - | 0xE2 0xB0 0x80..0xAE #L& [47] GLAGOLITIC CAPITAL LETTER AZU..GLA... - | 0xE2 0xB0 0xB0..0xFF #L& [47] GLAGOLITIC SMALL LETTER AZU..GLAGO... - | 0xE2 0xB1 0x00..0x9E # - | 0xE2 0xB1 0xA0..0xBB #L& [28] LATIN CAPITAL LETTER L WITH DOUBLE... - | 0xE2 0xB1 0xBC..0xBD #Lm [2] LATIN SUBSCRIPT SMALL LETTER J..MO... - | 0xE2 0xB1 0xBE..0xFF #L& [103] LATIN CAPITAL LETTER S WITH SW... - | 0xE2 0xB2..0xB2 0x00..0xFF # - | 0xE2 0xB3 0x00..0xA4 # - | 0xE2 0xB3 0xAB..0xAE #L& [4] COPTIC CAPITAL LETTER CRYPTOGRAMMI... - | 0xE2 0xB3 0xB2..0xB3 #L& [2] COPTIC CAPITAL LETTER BOHAIRIC KHE... - | 0xE2 0xB4 0x80..0xA5 #L& [38] GEORGIAN SMALL LETTER AN..GEORGIAN... - | 0xE2 0xB4 0xA7 #L& GEORGIAN SMALL LETTER YN - | 0xE2 0xB4 0xAD #L& GEORGIAN SMALL LETTER AEN - | 0xE2 0xB4 0xB0..0xFF #Lo [56] TIFINAGH LETTER YA..TIFINAGH LETTE... - | 0xE2 0xB5 0x00..0xA7 # - | 0xE2 0xB5 0xAF #Lm TIFINAGH MODIFIER LETTER LABIALIZA... - | 0xE2 0xB6 0x80..0x96 #Lo [23] ETHIOPIC SYLLABLE LOA..ETHIOPIC SY... - | 0xE2 0xB6 0xA0..0xA6 #Lo [7] ETHIOPIC SYLLABLE SSA..ETHIOPIC SY... - | 0xE2 0xB6 0xA8..0xAE #Lo [7] ETHIOPIC SYLLABLE CCA..ETHIOPIC SY... - | 0xE2 0xB6 0xB0..0xB6 #Lo [7] ETHIOPIC SYLLABLE ZZA..ETHIOPIC SY... - | 0xE2 0xB6 0xB8..0xBE #Lo [7] ETHIOPIC SYLLABLE CCHA..ETHIOPIC S... - | 0xE2 0xB7 0x80..0x86 #Lo [7] ETHIOPIC SYLLABLE QYA..ETHIOPIC SY... - | 0xE2 0xB7 0x88..0x8E #Lo [7] ETHIOPIC SYLLABLE KYA..ETHIOPIC SY... - | 0xE2 0xB7 0x90..0x96 #Lo [7] ETHIOPIC SYLLABLE XYA..ETHIOPIC SY... - | 0xE2 0xB7 0x98..0x9E #Lo [7] ETHIOPIC SYLLABLE GYA..ETHIOPIC SY... - | 0xE2 0xB8 0xAF #Lm VERTICAL TILDE - | 0xE3 0x80 0x85 #Lm IDEOGRAPHIC ITERATION MARK - | 0xE3 0x80 0xBB #Lm VERTICAL IDEOGRAPHIC ITERATION MARK - | 0xE3 0x80 0xBC #Lo MASU MARK - | 0xE3 0x84 0x85..0xAD #Lo [41] BOPOMOFO LETTER B..BOPOMOFO LETTER IH - | 0xE3 0x84 0xB1..0xFF #Lo [94] HANGUL LETTER KIYEOK..HANGUL L... - | 0xE3 0x85..0x85 0x00..0xFF # - | 0xE3 0x86 0x00..0x8E # - | 0xE3 0x86 0xA0..0xBA #Lo [27] BOPOMOFO LETTER BU..BOPOMOFO LETTE... - | 0xEA 0x80 0x80..0x94 #Lo [21] YI SYLLABLE IT..YI SYLLABLE E - | 0xEA 0x80 0x95 #Lm YI SYLLABLE WU - | 0xEA 0x80 0x96..0xFF #Lo [1143] YI SYLLABLE BIT..YI SYLLABLE YYR - | 0xEA 0x81..0x91 0x00..0xFF # - | 0xEA 0x92 0x00..0x8C # - | 0xEA 0x93 0x90..0xB7 #Lo [40] LISU LETTER BA..LISU LETTER OE - | 0xEA 0x93 0xB8..0xBD #Lm [6] LISU LETTER TONE MYA TI..LISU LETT... - | 0xEA 0x94 0x80..0xFF #Lo [268] VAI SYLLABLE EE..VAI SYLLABLE NG - | 0xEA 0x95..0x97 0x00..0xFF # - | 0xEA 0x98 0x00..0x8B # - | 0xEA 0x98 0x8C #Lm VAI SYLLABLE LENGTHENER - | 0xEA 0x98 0x90..0x9F #Lo [16] VAI SYLLABLE NDOLE FA..VAI SYMBOL ... - | 0xEA 0x98 0xAA..0xAB #Lo [2] VAI SYLLABLE NDOLE MA..VAI SYLLABL... - | 0xEA 0x99 0x80..0xAD #L& [46] CYRILLIC CAPITAL LETTER ZEMLYA..CY... - | 0xEA 0x99 0xAE #Lo CYRILLIC LETTER MULTIOCULAR O - | 0xEA 0x99 0xBF #Lm CYRILLIC PAYEROK - | 0xEA 0x9A 0x80..0x9B #L& [28] CYRILLIC CAPITAL LETTER DWE..CYRIL... - | 0xEA 0x9A 0x9C..0x9D #Lm [2] MODIFIER LETTER CYRILLIC HARD SIGN... - | 0xEA 0x9A 0xA0..0xFF #Lo [70] BAMUM LETTER A..BAMUM LETTER KI - | 0xEA 0x9B 0x00..0xA5 # - | 0xEA 0x9B 0xA6..0xAF #Nl [10] BAMUM LETTER MO..BAMUM LETTER KOGHOM - | 0xEA 0x9C 0x97..0x9F #Lm [9] MODIFIER LETTER DOT VERTICAL BAR..... - | 0xEA 0x9C 0xA2..0xFF #L& [78] LATIN CAPITAL LETTER EGYPTOLOGICAL... - | 0xEA 0x9D 0x00..0xAF # - | 0xEA 0x9D 0xB0 #Lm MODIFIER LETTER US - | 0xEA 0x9D 0xB1..0xFF #L& [23] LATIN SMALL LETTER DUM..LATIN SMAL... - | 0xEA 0x9E 0x00..0x87 # - | 0xEA 0x9E 0x88 #Lm MODIFIER LETTER LOW CIRCUMFLEX ACCENT - | 0xEA 0x9E 0x8B..0x8E #L& [4] LATIN CAPITAL LETTER SALTILLO..LAT... - | 0xEA 0x9E 0x8F #Lo LATIN LETTER SINOLOGICAL DOT - | 0xEA 0x9E 0x90..0xAD #L& [30] LATIN CAPITAL LETTER N WITH DESCEN... - | 0xEA 0x9E 0xB0..0xB7 #L& [8] LATIN CAPITAL LETTER TURNED K..LAT... - | 0xEA 0x9F 0xB7 #Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I - | 0xEA 0x9F 0xB8..0xB9 #Lm [2] MODIFIER LETTER CAPITAL H WITH STR... - | 0xEA 0x9F 0xBA #L& LATIN LETTER SMALL CAPITAL TURNED M - | 0xEA 0x9F 0xBB..0xFF #Lo [7] LATIN EPIGRAPHIC LETTER REVERSED F... - | 0xEA 0xA0 0x00..0x81 # - | 0xEA 0xA0 0x83..0x85 #Lo [3] SYLOTI NAGRI LETTER U..SYLOTI NAGR... - | 0xEA 0xA0 0x87..0x8A #Lo [4] SYLOTI NAGRI LETTER KO..SYLOTI NAG... - | 0xEA 0xA0 0x8C..0xA2 #Lo [23] SYLOTI NAGRI LETTER CO..SYLOTI NAG... - | 0xEA 0xA1 0x80..0xB3 #Lo [52] PHAGS-PA LETTER KA..PHAGS-PA LETTE... - | 0xEA 0xA2 0x82..0xB3 #Lo [50] SAURASHTRA LETTER A..SAURASHTRA LE... - | 0xEA 0xA3 0xB2..0xB7 #Lo [6] DEVANAGARI SIGN SPACING CANDRABIND... - | 0xEA 0xA3 0xBB #Lo DEVANAGARI HEADSTROKE - | 0xEA 0xA3 0xBD #Lo DEVANAGARI JAIN OM - | 0xEA 0xA4 0x8A..0xA5 #Lo [28] KAYAH LI LETTER KA..KAYAH LI LETTE... - | 0xEA 0xA4 0xB0..0xFF #Lo [23] REJANG LETTER KA..REJANG LETTER A - | 0xEA 0xA5 0x00..0x86 # - | 0xEA 0xA5 0xA0..0xBC #Lo [29] HANGUL CHOSEONG TIKEUT-MIEUM..HANG... - | 0xEA 0xA6 0x84..0xB2 #Lo [47] JAVANESE LETTER A..JAVANESE LETTER HA - | 0xEA 0xA7 0x8F #Lm JAVANESE PANGRANGKEP - | 0xEA 0xA8 0x80..0xA8 #Lo [41] CHAM LETTER A..CHAM LETTER HA - | 0xEA 0xA9 0x80..0x82 #Lo [3] CHAM LETTER FINAL K..CHAM LETTER F... - | 0xEA 0xA9 0x84..0x8B #Lo [8] CHAM LETTER FINAL CH..CHAM LETTER ... - | 0xEA 0xAB 0xA0..0xAA #Lo [11] MEETEI MAYEK LETTER E..MEETEI MAYE... - | 0xEA 0xAB 0xB2 #Lo MEETEI MAYEK ANJI - | 0xEA 0xAB 0xB3..0xB4 #Lm [2] MEETEI MAYEK SYLLABLE REPETITION M... - | 0xEA 0xAC 0x81..0x86 #Lo [6] ETHIOPIC SYLLABLE TTHU..ETHIOPIC S... - | 0xEA 0xAC 0x89..0x8E #Lo [6] ETHIOPIC SYLLABLE DDHU..ETHIOPIC S... - | 0xEA 0xAC 0x91..0x96 #Lo [6] ETHIOPIC SYLLABLE DZU..ETHIOPIC SY... - | 0xEA 0xAC 0xA0..0xA6 #Lo [7] ETHIOPIC SYLLABLE CCHHA..ETHIOPIC ... - | 0xEA 0xAC 0xA8..0xAE #Lo [7] ETHIOPIC SYLLABLE BBA..ETHIOPIC SY... - | 0xEA 0xAC 0xB0..0xFF #L& [43] LATIN SMALL LETTER BARRED ALPHA..L... - | 0xEA 0xAD 0x00..0x9A # - | 0xEA 0xAD 0x9C..0x9F #Lm [4] MODIFIER LETTER SMALL HENG..MODIFI... - | 0xEA 0xAD 0xA0..0xA5 #L& [6] LATIN SMALL LETTER SAKHA YAT..GREE... - | 0xEA 0xAD 0xB0..0xFF #L& [80] CHEROKEE SMALL LETTER A..CHEROKEE ... - | 0xEA 0xAE 0x00..0xBF # - | 0xEA 0xAF 0x80..0xA2 #Lo [35] MEETEI MAYEK LETTER KOK..MEETEI MA... - | 0xEA 0xB0 0x80..0xFF #Lo [11172] HANGUL SYLLABLE GA..HA... - | 0xEA 0xB1..0xFF 0x00..0xFF # - | 0xEB..0xEC 0x00..0xFF 0x00..0xFF # - | 0xED 0x00 0x00..0xFF # - | 0xED 0x01..0x9D 0x00..0xFF # - | 0xED 0x9E 0x00..0xA3 # - | 0xED 0x9E 0xB0..0xFF #Lo [23] HANGUL JUNGSEONG O-YEO..HANGUL JUN... - | 0xED 0x9F 0x00..0x86 # - | 0xED 0x9F 0x8B..0xBB #Lo [49] HANGUL JONGSEONG NIEUN-RIEUL..HANG... - | 0xEF 0xAC 0x80..0x86 #L& [7] LATIN SMALL LIGATURE FF..LATIN SMA... - | 0xEF 0xAC 0x93..0x97 #L& [5] ARMENIAN SMALL LIGATURE MEN NOW..A... - | 0xEF 0xAD 0x90..0xFF #Lo [98] ARABIC LETTER ALEF WASLA ISOLATED ... - | 0xEF 0xAE 0x00..0xB1 # - | 0xEF 0xAF 0x93..0xFF #Lo [363] ARABIC LETTER NG ISOLATED FORM... - | 0xEF 0xB0..0xB3 0x00..0xFF # - | 0xEF 0xB4 0x00..0xBD # - | 0xEF 0xB5 0x90..0xFF #Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH... - | 0xEF 0xB6 0x00..0x8F # - | 0xEF 0xB6 0x92..0xFF #Lo [54] ARABIC LIGATURE MEEM WITH JEEM WIT... - | 0xEF 0xB7 0x00..0x87 # - | 0xEF 0xB7 0xB0..0xBB #Lo [12] ARABIC LIGATURE SALLA USED AS KORA... - | 0xEF 0xB9 0xB0..0xB4 #Lo [5] ARABIC FATHATAN ISOLATED FORM..ARA... - | 0xEF 0xB9 0xB6..0xFF #Lo [135] ARABIC FATHA ISOLATED FORM..AR... - | 0xEF 0xBA..0xBA 0x00..0xFF # - | 0xEF 0xBB 0x00..0xBC # - | 0xEF 0xBC 0xA1..0xBA #L& [26] FULLWIDTH LATIN CAPITAL LETTER A..... - | 0xEF 0xBD 0x81..0x9A #L& [26] FULLWIDTH LATIN SMALL LETTER A..FU... - | 0xEF 0xBE 0xA0..0xBE #Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH... - | 0xEF 0xBF 0x82..0x87 #Lo [6] HALFWIDTH HANGUL LETTER A..HALFWID... - | 0xEF 0xBF 0x8A..0x8F #Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFW... - | 0xEF 0xBF 0x92..0x97 #Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWI... - | 0xEF 0xBF 0x9A..0x9C #Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWI... - | 0xF0 0x90 0x80 0x80..0x8B #Lo [12] LINEAR B SYLLABLE B008 A..LINEA... - | 0xF0 0x90 0x80 0x8D..0xA6 #Lo [26] LINEAR B SYLLABLE B036 JO..LINE... - | 0xF0 0x90 0x80 0xA8..0xBA #Lo [19] LINEAR B SYLLABLE B060 RA..LINE... - | 0xF0 0x90 0x80 0xBC..0xBD #Lo [2] LINEAR B SYLLABLE B017 ZA..LINE... - | 0xF0 0x90 0x80 0xBF..0xFF #Lo [15] LINEAR B SYLLABLE B020 ZO..LINE... - | 0xF0 0x90 0x81 0x00..0x8D # - | 0xF0 0x90 0x81 0x90..0x9D #Lo [14] LINEAR B SYMBOL B018..LINEAR B ... - | 0xF0 0x90 0x82 0x80..0xFF #Lo [123] LINEAR B IDEOGRAM B100 MAN..LIN... - | 0xF0 0x90 0x83 0x00..0xBA # - | 0xF0 0x90 0x85 0x80..0xB4 #Nl [53] GREEK ACROPHONIC ATTIC ONE QUAR... - | 0xF0 0x90 0x8A 0x80..0x9C #Lo [29] LYCIAN LETTER A..LYCIAN LETTER X - | 0xF0 0x90 0x8A 0xA0..0xFF #Lo [49] CARIAN LETTER A..CARIAN LETTER ... - | 0xF0 0x90 0x8B 0x00..0x90 # - | 0xF0 0x90 0x8C 0x80..0x9F #Lo [32] OLD ITALIC LETTER A..OLD ITALIC... - | 0xF0 0x90 0x8C 0xB0..0xFF #Lo [17] GOTHIC LETTER AHSA..GOTHIC LETT... - | 0xF0 0x90 0x8D 0x00..0x80 # - | 0xF0 0x90 0x8D 0x81 #Nl GOTHIC LETTER NINETY - | 0xF0 0x90 0x8D 0x82..0x89 #Lo [8] GOTHIC LETTER RAIDA..GOTHIC LET... - | 0xF0 0x90 0x8D 0x8A #Nl GOTHIC LETTER NINE HUNDRED - | 0xF0 0x90 0x8D 0x90..0xB5 #Lo [38] OLD PERMIC LETTER AN..OLD PERMI... - | 0xF0 0x90 0x8E 0x80..0x9D #Lo [30] UGARITIC LETTER ALPA..UGARITIC ... - | 0xF0 0x90 0x8E 0xA0..0xFF #Lo [36] OLD PERSIAN SIGN A..OLD PERSIAN... - | 0xF0 0x90 0x8F 0x00..0x83 # - | 0xF0 0x90 0x8F 0x88..0x8F #Lo [8] OLD PERSIAN SIGN AURAMAZDAA..OL... - | 0xF0 0x90 0x8F 0x91..0x95 #Nl [5] OLD PERSIAN NUMBER ONE..OLD PER... - | 0xF0 0x90 0x90 0x80..0xFF #L& [80] DESERET CAPITAL LETTER LONG I..... - | 0xF0 0x90 0x91 0x00..0x8F # - | 0xF0 0x90 0x91 0x90..0xFF #Lo [78] SHAVIAN LETTER PEEP..OSMANYA LE... - | 0xF0 0x90 0x92 0x00..0x9D # - | 0xF0 0x90 0x94 0x80..0xA7 #Lo [40] ELBASAN LETTER A..ELBASAN LETTE... - | 0xF0 0x90 0x94 0xB0..0xFF #Lo [52] CAUCASIAN ALBANIAN LETTER ALT..... - | 0xF0 0x90 0x95 0x00..0xA3 # - | 0xF0 0x90 0x98 0x80..0xFF #Lo [311] LINEAR A SIGN AB001..LINE... - | 0xF0 0x90 0x99..0x9B 0x00..0xFF # - | 0xF0 0x90 0x9C 0x00..0xB6 # - | 0xF0 0x90 0x9D 0x80..0x95 #Lo [22] LINEAR A SIGN A701 A..LINEAR A ... - | 0xF0 0x90 0x9D 0xA0..0xA7 #Lo [8] LINEAR A SIGN A800..LINEAR A SI... - | 0xF0 0x90 0xA0 0x80..0x85 #Lo [6] CYPRIOT SYLLABLE A..CYPRIOT SYL... - | 0xF0 0x90 0xA0 0x88 #Lo CYPRIOT SYLLABLE JO - | 0xF0 0x90 0xA0 0x8A..0xB5 #Lo [44] CYPRIOT SYLLABLE KA..CYPRIOT SY... - | 0xF0 0x90 0xA0 0xB7..0xB8 #Lo [2] CYPRIOT SYLLABLE XA..CYPRIOT SY... - | 0xF0 0x90 0xA0 0xBC #Lo CYPRIOT SYLLABLE ZA - | 0xF0 0x90 0xA0 0xBF..0xFF #Lo [23] CYPRIOT SYLLABLE ZO..IMPERIAL A... - | 0xF0 0x90 0xA1 0x00..0x95 # - | 0xF0 0x90 0xA1 0xA0..0xB6 #Lo [23] PALMYRENE LETTER ALEPH..PALMYRE... - | 0xF0 0x90 0xA2 0x80..0x9E #Lo [31] NABATAEAN LETTER FINAL ALEPH..N... - | 0xF0 0x90 0xA3 0xA0..0xB2 #Lo [19] HATRAN LETTER ALEPH..HATRAN LET... - | 0xF0 0x90 0xA3 0xB4..0xB5 #Lo [2] HATRAN LETTER SHIN..HATRAN LETT... - | 0xF0 0x90 0xA4 0x80..0x95 #Lo [22] PHOENICIAN LETTER ALF..PHOENICI... - | 0xF0 0x90 0xA4 0xA0..0xB9 #Lo [26] LYDIAN LETTER A..LYDIAN LETTER C - | 0xF0 0x90 0xA6 0x80..0xB7 #Lo [56] MEROITIC HIEROGLYPHIC LETTER A.... - | 0xF0 0x90 0xA6 0xBE..0xBF #Lo [2] MEROITIC CURSIVE LOGOGRAM RMT..... - | 0xF0 0x90 0xA8 0x80 #Lo KHAROSHTHI LETTER A - | 0xF0 0x90 0xA8 0x90..0x93 #Lo [4] KHAROSHTHI LETTER KA..KHAROSHTH... - | 0xF0 0x90 0xA8 0x95..0x97 #Lo [3] KHAROSHTHI LETTER CA..KHAROSHTH... - | 0xF0 0x90 0xA8 0x99..0xB3 #Lo [27] KHAROSHTHI LETTER NYA..KHAROSHT... - | 0xF0 0x90 0xA9 0xA0..0xBC #Lo [29] OLD SOUTH ARABIAN LETTER HE..OL... - | 0xF0 0x90 0xAA 0x80..0x9C #Lo [29] OLD NORTH ARABIAN LETTER HEH..O... - | 0xF0 0x90 0xAB 0x80..0x87 #Lo [8] MANICHAEAN LETTER ALEPH..MANICH... - | 0xF0 0x90 0xAB 0x89..0xA4 #Lo [28] MANICHAEAN LETTER ZAYIN..MANICH... - | 0xF0 0x90 0xAC 0x80..0xB5 #Lo [54] AVESTAN LETTER A..AVESTAN LETTE... - | 0xF0 0x90 0xAD 0x80..0x95 #Lo [22] INSCRIPTIONAL PARTHIAN LETTER A... - | 0xF0 0x90 0xAD 0xA0..0xB2 #Lo [19] INSCRIPTIONAL PAHLAVI LETTER AL... - | 0xF0 0x90 0xAE 0x80..0x91 #Lo [18] PSALTER PAHLAVI LETTER ALEPH..P... - | 0xF0 0x90 0xB0 0x80..0xFF #Lo [73] OLD TURKIC LETTER ORKHON A..OLD... - | 0xF0 0x90 0xB1 0x00..0x88 # - | 0xF0 0x90 0xB2 0x80..0xB2 #L& [51] OLD HUNGARIAN CAPITAL LETTER A.... - | 0xF0 0x90 0xB3 0x80..0xB2 #L& [51] OLD HUNGARIAN SMALL LETTER A..O... - | 0xF0 0x91 0x80 0x83..0xB7 #Lo [53] BRAHMI SIGN JIHVAMULIYA..BRAHMI... - | 0xF0 0x91 0x82 0x83..0xAF #Lo [45] KAITHI LETTER A..KAITHI LETTER HA - | 0xF0 0x91 0x83 0x90..0xA8 #Lo [25] SORA SOMPENG LETTER SAH..SORA S... - | 0xF0 0x91 0x84 0x83..0xA6 #Lo [36] CHAKMA LETTER AA..CHAKMA LETTER... - | 0xF0 0x91 0x85 0x90..0xB2 #Lo [35] MAHAJANI LETTER A..MAHAJANI LET... - | 0xF0 0x91 0x85 0xB6 #Lo MAHAJANI LIGATURE SHRI - | 0xF0 0x91 0x86 0x83..0xB2 #Lo [48] SHARADA LETTER A..SHARADA LETTE... - | 0xF0 0x91 0x87 0x81..0x84 #Lo [4] SHARADA SIGN AVAGRAHA..SHARADA OM - | 0xF0 0x91 0x87 0x9A #Lo SHARADA EKAM - | 0xF0 0x91 0x87 0x9C #Lo SHARADA HEADSTROKE - | 0xF0 0x91 0x88 0x80..0x91 #Lo [18] KHOJKI LETTER A..KHOJKI LETTER JJA - | 0xF0 0x91 0x88 0x93..0xAB #Lo [25] KHOJKI LETTER NYA..KHOJKI LETTE... - | 0xF0 0x91 0x8A 0x80..0x86 #Lo [7] MULTANI LETTER A..MULTANI LETTE... - | 0xF0 0x91 0x8A 0x88 #Lo MULTANI LETTER GHA - | 0xF0 0x91 0x8A 0x8A..0x8D #Lo [4] MULTANI LETTER CA..MULTANI LETT... - | 0xF0 0x91 0x8A 0x8F..0x9D #Lo [15] MULTANI LETTER NYA..MULTANI LET... - | 0xF0 0x91 0x8A 0x9F..0xA8 #Lo [10] MULTANI LETTER BHA..MULTANI LET... - | 0xF0 0x91 0x8A 0xB0..0xFF #Lo [47] KHUDAWADI LETTER A..KHUDAWADI L... - | 0xF0 0x91 0x8B 0x00..0x9E # - | 0xF0 0x91 0x8C 0x85..0x8C #Lo [8] GRANTHA LETTER A..GRANTHA LETTE... - | 0xF0 0x91 0x8C 0x8F..0x90 #Lo [2] GRANTHA LETTER EE..GRANTHA LETT... - | 0xF0 0x91 0x8C 0x93..0xA8 #Lo [22] GRANTHA LETTER OO..GRANTHA LETT... - | 0xF0 0x91 0x8C 0xAA..0xB0 #Lo [7] GRANTHA LETTER PA..GRANTHA LETT... - | 0xF0 0x91 0x8C 0xB2..0xB3 #Lo [2] GRANTHA LETTER LA..GRANTHA LETT... - | 0xF0 0x91 0x8C 0xB5..0xB9 #Lo [5] GRANTHA LETTER VA..GRANTHA LETT... - | 0xF0 0x91 0x8C 0xBD #Lo GRANTHA SIGN AVAGRAHA - | 0xF0 0x91 0x8D 0x90 #Lo GRANTHA OM - | 0xF0 0x91 0x8D 0x9D..0xA1 #Lo [5] GRANTHA SIGN PLUTA..GRANTHA LET... - | 0xF0 0x91 0x92 0x80..0xAF #Lo [48] TIRHUTA ANJI..TIRHUTA LETTER HA - | 0xF0 0x91 0x93 0x84..0x85 #Lo [2] TIRHUTA SIGN AVAGRAHA..TIRHUTA ... - | 0xF0 0x91 0x93 0x87 #Lo TIRHUTA OM - | 0xF0 0x91 0x96 0x80..0xAE #Lo [47] SIDDHAM LETTER A..SIDDHAM LETTE... - | 0xF0 0x91 0x97 0x98..0x9B #Lo [4] SIDDHAM LETTER THREE-CIRCLE ALT... - | 0xF0 0x91 0x98 0x80..0xAF #Lo [48] MODI LETTER A..MODI LETTER LLA - | 0xF0 0x91 0x99 0x84 #Lo MODI SIGN HUVA - | 0xF0 0x91 0x9A 0x80..0xAA #Lo [43] TAKRI LETTER A..TAKRI LETTER RRA - | 0xF0 0x91 0xA2 0xA0..0xFF #L& [64] WARANG CITI CAPITAL LETTER NGAA... - | 0xF0 0x91 0xA3 0x00..0x9F # - | 0xF0 0x91 0xA3 0xBF #Lo WARANG CITI OM - | 0xF0 0x91 0xAB 0x80..0xB8 #Lo [57] PAU CIN HAU LETTER PA..PAU CIN ... - | 0xF0 0x92 0x80 0x80..0xFF #Lo [922] CUNEIFORM SIGN A..CUNEIFO... - | 0xF0 0x92 0x81..0x8D 0x00..0xFF # - | 0xF0 0x92 0x8E 0x00..0x99 # - | 0xF0 0x92 0x90 0x80..0xFF #Nl [111] CUNEIFORM NUMERIC SIGN TWO ASH.... - | 0xF0 0x92 0x91 0x00..0xAE # - | 0xF0 0x92 0x92 0x80..0xFF #Lo [196] CUNEIFORM SIGN AB TIMES N... - | 0xF0 0x92 0x93..0x94 0x00..0xFF # - | 0xF0 0x92 0x95 0x00..0x83 # - | 0xF0 0x93 0x80 0x80..0xFF #Lo [1071] EGYPTIAN HIEROGLYPH A001... - | 0xF0 0x93 0x81..0x8F 0x00..0xFF # - | 0xF0 0x93 0x90 0x00..0xAE # - | 0xF0 0x94 0x90 0x80..0xFF #Lo [583] ANATOLIAN HIEROGLYPH A001... - | 0xF0 0x94 0x91..0x98 0x00..0xFF # - | 0xF0 0x94 0x99 0x00..0x86 # - | 0xF0 0x96 0xA0 0x80..0xFF #Lo [569] BAMUM LETTER PHASE-A NGKU... - | 0xF0 0x96 0xA1..0xA7 0x00..0xFF # - | 0xF0 0x96 0xA8 0x00..0xB8 # - | 0xF0 0x96 0xA9 0x80..0x9E #Lo [31] MRO LETTER TA..MRO LETTER TEK - | 0xF0 0x96 0xAB 0x90..0xAD #Lo [30] BASSA VAH LETTER ENNI..BASSA VA... - | 0xF0 0x96 0xAC 0x80..0xAF #Lo [48] PAHAWH HMONG VOWEL KEEB..PAHAWH... - | 0xF0 0x96 0xAD 0x80..0x83 #Lm [4] PAHAWH HMONG SIGN VOS SEEV..PAH... - | 0xF0 0x96 0xAD 0xA3..0xB7 #Lo [21] PAHAWH HMONG SIGN VOS LUB..PAHA... - | 0xF0 0x96 0xAD 0xBD..0xFF #Lo [19] PAHAWH HMONG CLAN SIGN TSHEEJ..... - | 0xF0 0x96 0xAE 0x00..0x8F # - | 0xF0 0x96 0xBC 0x80..0xFF #Lo [69] MIAO LETTER PA..MIAO LETTER HHA - | 0xF0 0x96 0xBD 0x00..0x84 # - | 0xF0 0x96 0xBD 0x90 #Lo MIAO LETTER NASALIZATION - | 0xF0 0x96 0xBE 0x93..0x9F #Lm [13] MIAO LETTER TONE-2..MIAO LETTER... - | 0xF0 0x9B 0xB0 0x80..0xFF #Lo [107] DUPLOYAN LETTER H..DUPLOYAN LET... - | 0xF0 0x9B 0xB1 0x00..0xAA # - | 0xF0 0x9B 0xB1 0xB0..0xBC #Lo [13] DUPLOYAN AFFIX LEFT HORIZONTAL ... - | 0xF0 0x9B 0xB2 0x80..0x88 #Lo [9] DUPLOYAN AFFIX HIGH ACUTE..DUPL... - | 0xF0 0x9B 0xB2 0x90..0x99 #Lo [10] DUPLOYAN AFFIX LOW ACUTE..DUPLO... - | 0xF0 0x9D 0x90 0x80..0xFF #L& [85] MATHEMATICAL BOLD CAPITAL A..MA... - | 0xF0 0x9D 0x91 0x00..0x94 # - | 0xF0 0x9D 0x91 0x96..0xFF #L& [71] MATHEMATICAL ITALIC SMALL I..MA... - | 0xF0 0x9D 0x92 0x00..0x9C # - | 0xF0 0x9D 0x92 0x9E..0x9F #L& [2] MATHEMATICAL SCRIPT CAPITAL C..... - | 0xF0 0x9D 0x92 0xA2 #L& MATHEMATICAL SCRIPT CAPITAL G - | 0xF0 0x9D 0x92 0xA5..0xA6 #L& [2] MATHEMATICAL SCRIPT CAPITAL J..... - | 0xF0 0x9D 0x92 0xA9..0xAC #L& [4] MATHEMATICAL SCRIPT CAPITAL N..... - | 0xF0 0x9D 0x92 0xAE..0xB9 #L& [12] MATHEMATICAL SCRIPT CAPITAL S..... - | 0xF0 0x9D 0x92 0xBB #L& MATHEMATICAL SCRIPT SMALL F - | 0xF0 0x9D 0x92 0xBD..0xFF #L& [7] MATHEMATICAL SCRIPT SMALL H..MA... - | 0xF0 0x9D 0x93 0x00..0x83 # - | 0xF0 0x9D 0x93 0x85..0xFF #L& [65] MATHEMATICAL SCRIPT SMALL P..MA... - | 0xF0 0x9D 0x94 0x00..0x85 # - | 0xF0 0x9D 0x94 0x87..0x8A #L& [4] MATHEMATICAL FRAKTUR CAPITAL D.... - | 0xF0 0x9D 0x94 0x8D..0x94 #L& [8] MATHEMATICAL FRAKTUR CAPITAL J.... - | 0xF0 0x9D 0x94 0x96..0x9C #L& [7] MATHEMATICAL FRAKTUR CAPITAL S.... - | 0xF0 0x9D 0x94 0x9E..0xB9 #L& [28] MATHEMATICAL FRAKTUR SMALL A..M... - | 0xF0 0x9D 0x94 0xBB..0xBE #L& [4] MATHEMATICAL DOUBLE-STRUCK CAPI... - | 0xF0 0x9D 0x95 0x80..0x84 #L& [5] MATHEMATICAL DOUBLE-STRUCK CAPI... - | 0xF0 0x9D 0x95 0x86 #L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O - | 0xF0 0x9D 0x95 0x8A..0x90 #L& [7] MATHEMATICAL DOUBLE-STRUCK CAPI... - | 0xF0 0x9D 0x95 0x92..0xFF #L& [340] MATHEMATICAL DOUBLE-STRUC... - | 0xF0 0x9D 0x96..0x99 0x00..0xFF # - | 0xF0 0x9D 0x9A 0x00..0xA5 # - | 0xF0 0x9D 0x9A 0xA8..0xFF #L& [25] MATHEMATICAL BOLD CAPITAL ALPHA... - | 0xF0 0x9D 0x9B 0x00..0x80 # - | 0xF0 0x9D 0x9B 0x82..0x9A #L& [25] MATHEMATICAL BOLD SMALL ALPHA..... - | 0xF0 0x9D 0x9B 0x9C..0xBA #L& [31] MATHEMATICAL BOLD EPSILON SYMBO... - | 0xF0 0x9D 0x9B 0xBC..0xFF #L& [25] MATHEMATICAL ITALIC SMALL ALPHA... - | 0xF0 0x9D 0x9C 0x00..0x94 # - | 0xF0 0x9D 0x9C 0x96..0xB4 #L& [31] MATHEMATICAL ITALIC EPSILON SYM... - | 0xF0 0x9D 0x9C 0xB6..0xFF #L& [25] MATHEMATICAL BOLD ITALIC SMALL ... - | 0xF0 0x9D 0x9D 0x00..0x8E # - | 0xF0 0x9D 0x9D 0x90..0xAE #L& [31] MATHEMATICAL BOLD ITALIC EPSILO... - | 0xF0 0x9D 0x9D 0xB0..0xFF #L& [25] MATHEMATICAL SANS-SERIF BOLD SM... - | 0xF0 0x9D 0x9E 0x00..0x88 # - | 0xF0 0x9D 0x9E 0x8A..0xA8 #L& [31] MATHEMATICAL SANS-SERIF BOLD EP... - | 0xF0 0x9D 0x9E 0xAA..0xFF #L& [25] MATHEMATICAL SANS-SERIF BOLD IT... - | 0xF0 0x9D 0x9F 0x00..0x82 # - | 0xF0 0x9D 0x9F 0x84..0x8B #L& [8] MATHEMATICAL SANS-SERIF BOLD IT... - | 0xF0 0x9E 0xA0 0x80..0xFF #Lo [197] MENDE KIKAKUI SYLLABLE M0... - | 0xF0 0x9E 0xA1..0xA2 0x00..0xFF # - | 0xF0 0x9E 0xA3 0x00..0x84 # - | 0xF0 0x9E 0xB8 0x80..0x83 #Lo [4] ARABIC MATHEMATICAL ALEF..ARABI... - | 0xF0 0x9E 0xB8 0x85..0x9F #Lo [27] ARABIC MATHEMATICAL WAW..ARABIC... - | 0xF0 0x9E 0xB8 0xA1..0xA2 #Lo [2] ARABIC MATHEMATICAL INITIAL BEH... - | 0xF0 0x9E 0xB8 0xA4 #Lo ARABIC MATHEMATICAL INITIAL HEH - | 0xF0 0x9E 0xB8 0xA7 #Lo ARABIC MATHEMATICAL INITIAL HAH - | 0xF0 0x9E 0xB8 0xA9..0xB2 #Lo [10] ARABIC MATHEMATICAL INITIAL YEH... - | 0xF0 0x9E 0xB8 0xB4..0xB7 #Lo [4] ARABIC MATHEMATICAL INITIAL SHE... - | 0xF0 0x9E 0xB8 0xB9 #Lo ARABIC MATHEMATICAL INITIAL DAD - | 0xF0 0x9E 0xB8 0xBB #Lo ARABIC MATHEMATICAL INITIAL GHAIN - | 0xF0 0x9E 0xB9 0x82 #Lo ARABIC MATHEMATICAL TAILED JEEM - | 0xF0 0x9E 0xB9 0x87 #Lo ARABIC MATHEMATICAL TAILED HAH - | 0xF0 0x9E 0xB9 0x89 #Lo ARABIC MATHEMATICAL TAILED YEH - | 0xF0 0x9E 0xB9 0x8B #Lo ARABIC MATHEMATICAL TAILED LAM - | 0xF0 0x9E 0xB9 0x8D..0x8F #Lo [3] ARABIC MATHEMATICAL TAILED NOON... - | 0xF0 0x9E 0xB9 0x91..0x92 #Lo [2] ARABIC MATHEMATICAL TAILED SAD.... - | 0xF0 0x9E 0xB9 0x94 #Lo ARABIC MATHEMATICAL TAILED SHEEN - | 0xF0 0x9E 0xB9 0x97 #Lo ARABIC MATHEMATICAL TAILED KHAH - | 0xF0 0x9E 0xB9 0x99 #Lo ARABIC MATHEMATICAL TAILED DAD - | 0xF0 0x9E 0xB9 0x9B #Lo ARABIC MATHEMATICAL TAILED GHAIN - | 0xF0 0x9E 0xB9 0x9D #Lo ARABIC MATHEMATICAL TAILED DOTLESS... - | 0xF0 0x9E 0xB9 0x9F #Lo ARABIC MATHEMATICAL TAILED DOTLESS... - | 0xF0 0x9E 0xB9 0xA1..0xA2 #Lo [2] ARABIC MATHEMATICAL STRETCHED B... - | 0xF0 0x9E 0xB9 0xA4 #Lo ARABIC MATHEMATICAL STRETCHED HEH - | 0xF0 0x9E 0xB9 0xA7..0xAA #Lo [4] ARABIC MATHEMATICAL STRETCHED H... - | 0xF0 0x9E 0xB9 0xAC..0xB2 #Lo [7] ARABIC MATHEMATICAL STRETCHED M... - | 0xF0 0x9E 0xB9 0xB4..0xB7 #Lo [4] ARABIC MATHEMATICAL STRETCHED S... - | 0xF0 0x9E 0xB9 0xB9..0xBC #Lo [4] ARABIC MATHEMATICAL STRETCHED D... - | 0xF0 0x9E 0xB9 0xBE #Lo ARABIC MATHEMATICAL STRETCHED DOTL... - | 0xF0 0x9E 0xBA 0x80..0x89 #Lo [10] ARABIC MATHEMATICAL LOOPED ALEF... - | 0xF0 0x9E 0xBA 0x8B..0x9B #Lo [17] ARABIC MATHEMATICAL LOOPED LAM.... - | 0xF0 0x9E 0xBA 0xA1..0xA3 #Lo [3] ARABIC MATHEMATICAL DOUBLE-STRU... - | 0xF0 0x9E 0xBA 0xA5..0xA9 #Lo [5] ARABIC MATHEMATICAL DOUBLE-STRU... - | 0xF0 0x9E 0xBA 0xAB..0xBB #Lo [17] ARABIC MATHEMATICAL DOUBLE-STRU... - | 0xF0 0x9F 0x84 0xB0..0xFF #So [26] SQUARED LATIN CAPITAL LETTER A.... - | 0xF0 0x9F 0x85 0x00..0x89 # - | 0xF0 0x9F 0x85 0x90..0xA9 #So [26] NEGATIVE CIRCLED LATIN CAPITAL ... - | 0xF0 0x9F 0x85 0xB0..0xFF #So [26] NEGATIVE SQUARED LATIN CAPITAL ... - | 0xF0 0x9F 0x86 0x00..0x89 # - ; - - MidLetter = - 0x3A #Po COLON - | 0xC2 0xB7 #Po MIDDLE DOT - | 0xCB 0x97 #Sk MODIFIER LETTER MINUS SIGN - | 0xCE 0x87 #Po GREEK ANO TELEIA - | 0xD7 0xB4 #Po HEBREW PUNCTUATION GERSHAYIM - | 0xE2 0x80 0xA7 #Po HYPHENATION POINT - | 0xEF 0xB8 0x93 #Po PRESENTATION FORM FOR VERTICAL COLON - | 0xEF 0xB9 0x95 #Po SMALL COLON - | 0xEF 0xBC 0x9A #Po FULLWIDTH COLON - ; - - MidNum = - 0x2C #Po COMMA - | 0x3B #Po SEMICOLON - | 0xCD 0xBE #Po GREEK QUESTION MARK - | 0xD6 0x89 #Po ARMENIAN FULL STOP - | 0xD8 0x8C..0x8D #Po [2] ARABIC COMMA..ARABIC DATE SEPARATOR - | 0xD9 0xAC #Po ARABIC THOUSANDS SEPARATOR - | 0xDF 0xB8 #Po NKO COMMA - | 0xE2 0x81 0x84 #Sm FRACTION SLASH - | 0xEF 0xB8 0x90 #Po PRESENTATION FORM FOR VERTICAL COMMA - | 0xEF 0xB8 0x94 #Po PRESENTATION FORM FOR VERTICAL SEM... - | 0xEF 0xB9 0x90 #Po SMALL COMMA - | 0xEF 0xB9 0x94 #Po SMALL SEMICOLON - | 0xEF 0xBC 0x8C #Po FULLWIDTH COMMA - | 0xEF 0xBC 0x9B #Po FULLWIDTH SEMICOLON - ; - - MidNumLet = - 0x2E #Po FULL STOP - | 0xE2 0x80 0x98 #Pi LEFT SINGLE QUOTATION MARK - | 0xE2 0x80 0x99 #Pf RIGHT SINGLE QUOTATION MARK - | 0xE2 0x80 0xA4 #Po ONE DOT LEADER - | 0xEF 0xB9 0x92 #Po SMALL FULL STOP - | 0xEF 0xBC 0x87 #Po FULLWIDTH APOSTROPHE - | 0xEF 0xBC 0x8E #Po FULLWIDTH FULL STOP - ; - - Numeric = - 0x30..0x39 #Nd [10] DIGIT ZERO..DIGIT NINE - | 0xD9 0xA0..0xA9 #Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-IN... - | 0xD9 0xAB #Po ARABIC DECIMAL SEPARATOR - | 0xDB 0xB0..0xB9 #Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..... - | 0xDF 0x80..0x89 #Nd [10] NKO DIGIT ZERO..NKO DIGIT NINE - | 0xE0 0xA5 0xA6..0xAF #Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI ... - | 0xE0 0xA7 0xA6..0xAF #Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT ... - | 0xE0 0xA9 0xA6..0xAF #Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGI... - | 0xE0 0xAB 0xA6..0xAF #Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGI... - | 0xE0 0xAD 0xA6..0xAF #Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE - | 0xE0 0xAF 0xA6..0xAF #Nd [10] TAMIL DIGIT ZERO..TAMIL DIGIT NINE - | 0xE0 0xB1 0xA6..0xAF #Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE - | 0xE0 0xB3 0xA6..0xAF #Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT ... - | 0xE0 0xB5 0xA6..0xAF #Nd [10] MALAYALAM DIGIT ZERO..MALAYALAM DI... - | 0xE0 0xB7 0xA6..0xAF #Nd [10] SINHALA LITH DIGIT ZERO..SINHALA L... - | 0xE0 0xB9 0x90..0x99 #Nd [10] THAI DIGIT ZERO..THAI DIGIT NINE - | 0xE0 0xBB 0x90..0x99 #Nd [10] LAO DIGIT ZERO..LAO DIGIT NINE - | 0xE0 0xBC 0xA0..0xA9 #Nd [10] TIBETAN DIGIT ZERO..TIBETAN DIGIT ... - | 0xE1 0x81 0x80..0x89 #Nd [10] MYANMAR DIGIT ZERO..MYANMAR DIGIT ... - | 0xE1 0x82 0x90..0x99 #Nd [10] MYANMAR SHAN DIGIT ZERO..MYANMAR S... - | 0xE1 0x9F 0xA0..0xA9 #Nd [10] KHMER DIGIT ZERO..KHMER DIGIT NINE - | 0xE1 0xA0 0x90..0x99 #Nd [10] MONGOLIAN DIGIT ZERO..MONGOLIAN DI... - | 0xE1 0xA5 0x86..0x8F #Nd [10] LIMBU DIGIT ZERO..LIMBU DIGIT NINE - | 0xE1 0xA7 0x90..0x99 #Nd [10] NEW TAI LUE DIGIT ZERO..NEW TAI LU... - | 0xE1 0xAA 0x80..0x89 #Nd [10] TAI THAM HORA DIGIT ZERO..TAI THAM... - | 0xE1 0xAA 0x90..0x99 #Nd [10] TAI THAM THAM DIGIT ZERO..TAI THAM... - | 0xE1 0xAD 0x90..0x99 #Nd [10] BALINESE DIGIT ZERO..BALINESE DIGI... - | 0xE1 0xAE 0xB0..0xB9 #Nd [10] SUNDANESE DIGIT ZERO..SUNDANESE DI... - | 0xE1 0xB1 0x80..0x89 #Nd [10] LEPCHA DIGIT ZERO..LEPCHA DIGIT NINE - | 0xE1 0xB1 0x90..0x99 #Nd [10] OL CHIKI DIGIT ZERO..OL CHIKI DIGI... - | 0xEA 0x98 0xA0..0xA9 #Nd [10] VAI DIGIT ZERO..VAI DIGIT NINE - | 0xEA 0xA3 0x90..0x99 #Nd [10] SAURASHTRA DIGIT ZERO..SAURASHTRA ... - | 0xEA 0xA4 0x80..0x89 #Nd [10] KAYAH LI DIGIT ZERO..KAYAH LI DIGI... - | 0xEA 0xA7 0x90..0x99 #Nd [10] JAVANESE DIGIT ZERO..JAVANESE DIGI... - | 0xEA 0xA7 0xB0..0xB9 #Nd [10] MYANMAR TAI LAING DIGIT ZERO..MYAN... - | 0xEA 0xA9 0x90..0x99 #Nd [10] CHAM DIGIT ZERO..CHAM DIGIT NINE - | 0xEA 0xAF 0xB0..0xB9 #Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MA... - | 0xF0 0x90 0x92 0xA0..0xA9 #Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIG... - | 0xF0 0x91 0x81 0xA6..0xAF #Nd [10] BRAHMI DIGIT ZERO..BRAHMI DIGIT... - | 0xF0 0x91 0x83 0xB0..0xB9 #Nd [10] SORA SOMPENG DIGIT ZERO..SORA S... - | 0xF0 0x91 0x84 0xB6..0xBF #Nd [10] CHAKMA DIGIT ZERO..CHAKMA DIGIT... - | 0xF0 0x91 0x87 0x90..0x99 #Nd [10] SHARADA DIGIT ZERO..SHARADA DIG... - | 0xF0 0x91 0x8B 0xB0..0xB9 #Nd [10] KHUDAWADI DIGIT ZERO..KHUDAWADI... - | 0xF0 0x91 0x93 0x90..0x99 #Nd [10] TIRHUTA DIGIT ZERO..TIRHUTA DIG... - | 0xF0 0x91 0x99 0x90..0x99 #Nd [10] MODI DIGIT ZERO..MODI DIGIT NINE - | 0xF0 0x91 0x9B 0x80..0x89 #Nd [10] TAKRI DIGIT ZERO..TAKRI DIGIT NINE - | 0xF0 0x91 0x9C 0xB0..0xB9 #Nd [10] AHOM DIGIT ZERO..AHOM DIGIT NINE - | 0xF0 0x91 0xA3 0xA0..0xA9 #Nd [10] WARANG CITI DIGIT ZERO..WARANG ... - | 0xF0 0x96 0xA9 0xA0..0xA9 #Nd [10] MRO DIGIT ZERO..MRO DIGIT NINE - | 0xF0 0x96 0xAD 0x90..0x99 #Nd [10] PAHAWH HMONG DIGIT ZERO..PAHAWH... - | 0xF0 0x9D 0x9F 0x8E..0xBF #Nd [50] MATHEMATICAL BOLD DIGIT ZERO..M... - ; - - ExtendNumLet = - 0x5F #Pc LOW LINE - | 0xE2 0x80 0xBF..0xFF #Pc [2] UNDERTIE..CHARACTER TIE - | 0xE2 0x81 0x00..0x80 # - | 0xE2 0x81 0x94 #Pc INVERTED UNDERTIE - | 0xEF 0xB8 0xB3..0xB4 #Pc [2] PRESENTATION FORM FOR VERTICAL LOW... - | 0xEF 0xB9 0x8D..0x8F #Pc [3] DASHED LOW LINE..WAVY LOW LINE - | 0xEF 0xBC 0xBF #Pc FULLWIDTH LOW LINE - ; - - Regional_Indicator = - 0xF0 0x9F 0x87 0xA6..0xBF #So [26] REGIONAL INDICATOR SYMBOL LETTE... - ; - -}%% diff --git a/vendor/github.com/blevesearch/segment/segment_fuzz_test.go b/vendor/github.com/blevesearch/segment/segment_fuzz_test.go deleted file mode 100644 index 4fa5737a..00000000 --- a/vendor/github.com/blevesearch/segment/segment_fuzz_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -// +build gofuzz_generate - -package segment - -import ( - "io/ioutil" - "os" - "strconv" - "testing" -) - -const fuzzPrefix = "workdir/corpus" - -func TestGenerateWordSegmentFuzz(t *testing.T) { - - os.MkdirAll(fuzzPrefix, 0777) - for i, test := range unicodeWordTests { - ioutil.WriteFile(fuzzPrefix+"/"+strconv.Itoa(i)+".txt", test.input, 0777) - } -} diff --git a/vendor/github.com/blevesearch/segment/segment_test.go b/vendor/github.com/blevesearch/segment/segment_test.go deleted file mode 100644 index b799ef6c..00000000 --- a/vendor/github.com/blevesearch/segment/segment_test.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package segment - -import ( - "bufio" - "bytes" - "errors" - "io" - "strings" - "testing" -) - -// Tests borrowed from Scanner to test Segmenter - -// slowReader is a reader that returns only a few bytes at a time, to test the incremental -// reads in Scanner.Scan. -type slowReader struct { - max int - buf io.Reader -} - -func (sr *slowReader) Read(p []byte) (n int, err error) { - if len(p) > sr.max { - p = p[0:sr.max] - } - return sr.buf.Read(p) -} - -// genLine writes to buf a predictable but non-trivial line of text of length -// n, including the terminal newline and an occasional carriage return. -// If addNewline is false, the \r and \n are not emitted. -func genLine(buf *bytes.Buffer, lineNum, n int, addNewline bool) { - buf.Reset() - doCR := lineNum%5 == 0 - if doCR { - n-- - } - for i := 0; i < n-1; i++ { // Stop early for \n. - c := 'a' + byte(lineNum+i) - if c == '\n' || c == '\r' { // Don't confuse us. - c = 'N' - } - buf.WriteByte(c) - } - if addNewline { - if doCR { - buf.WriteByte('\r') - } - buf.WriteByte('\n') - } - return -} - -func wrapSplitFuncAsSegmentFuncForTesting(splitFunc bufio.SplitFunc) SegmentFunc { - return func(data []byte, atEOF bool) (advance int, token []byte, typ int, err error) { - typ = 0 - advance, token, err = splitFunc(data, atEOF) - return - } -} - -// Test that the line segmenter errors out on a long line. -func TestSegmentTooLong(t *testing.T) { - const smallMaxTokenSize = 256 // Much smaller for more efficient testing. - // Build a buffer of lots of line lengths up to but not exceeding smallMaxTokenSize. - tmp := new(bytes.Buffer) - buf := new(bytes.Buffer) - lineNum := 0 - j := 0 - for i := 0; i < 2*smallMaxTokenSize; i++ { - genLine(tmp, lineNum, j, true) - j++ - buf.Write(tmp.Bytes()) - lineNum++ - } - s := NewSegmenter(&slowReader{3, buf}) - // change to line segmenter for testing - s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines)) - s.MaxTokenSize(smallMaxTokenSize) - j = 0 - for lineNum := 0; s.Segment(); lineNum++ { - genLine(tmp, lineNum, j, false) - if j < smallMaxTokenSize { - j++ - } else { - j-- - } - line := tmp.Bytes() - if !bytes.Equal(s.Bytes(), line) { - t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line) - } - } - err := s.Err() - if err != ErrTooLong { - t.Fatalf("expected ErrTooLong; got %s", err) - } -} - -var testError = errors.New("testError") - -// Test the correct error is returned when the split function errors out. -func TestSegmentError(t *testing.T) { - // Create a split function that delivers a little data, then a predictable error. - numSplits := 0 - const okCount = 7 - errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF { - panic("didn't get enough data") - } - if numSplits >= okCount { - return 0, nil, testError - } - numSplits++ - return 1, data[0:1], nil - } - // Read the data. - const text = "abcdefghijklmnopqrstuvwxyz" - buf := strings.NewReader(text) - s := NewSegmenter(&slowReader{1, buf}) - // change to line segmenter for testing - s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit)) - var i int - for i = 0; s.Segment(); i++ { - if len(s.Bytes()) != 1 || text[i] != s.Bytes()[0] { - t.Errorf("#%d: expected %q got %q", i, text[i], s.Bytes()[0]) - } - } - // Check correct termination location and error. - if i != okCount { - t.Errorf("unexpected termination; expected %d tokens got %d", okCount, i) - } - err := s.Err() - if err != testError { - t.Fatalf("expected %q got %v", testError, err) - } -} - -// Test that Scan finishes if we have endless empty reads. -type endlessZeros struct{} - -func (endlessZeros) Read(p []byte) (int, error) { - return 0, nil -} - -func TestBadReader(t *testing.T) { - scanner := NewSegmenter(endlessZeros{}) - for scanner.Segment() { - t.Fatal("read should fail") - } - err := scanner.Err() - if err != io.ErrNoProgress { - t.Errorf("unexpected error: %v", err) - } -} - -func TestSegmentAdvanceNegativeError(t *testing.T) { - errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF { - panic("didn't get enough data") - } - return -1, data[0:1], nil - } - // Read the data. - const text = "abcdefghijklmnopqrstuvwxyz" - buf := strings.NewReader(text) - s := NewSegmenter(&slowReader{1, buf}) - // change to line segmenter for testing - s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit)) - s.Segment() - err := s.Err() - if err != ErrNegativeAdvance { - t.Fatalf("expected %q got %v", testError, err) - } -} - -func TestSegmentAdvanceTooFarError(t *testing.T) { - errorSplit := func(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF { - panic("didn't get enough data") - } - return len(data) + 10, data[0:1], nil - } - // Read the data. - const text = "abcdefghijklmnopqrstuvwxyz" - buf := strings.NewReader(text) - s := NewSegmenter(&slowReader{1, buf}) - // change to line segmenter for testing - s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(errorSplit)) - s.Segment() - err := s.Err() - if err != ErrAdvanceTooFar { - t.Fatalf("expected %q got %v", testError, err) - } -} - -func TestSegmentLongTokens(t *testing.T) { - // Read the data. - text := bytes.Repeat([]byte("abcdefghijklmnop"), 257) - buf := strings.NewReader(string(text)) - s := NewSegmenter(&slowReader{1, buf}) - // change to line segmenter for testing - s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines)) - for s.Segment() { - line := s.Bytes() - if !bytes.Equal(text, line) { - t.Errorf("expected %s, got %s", text, line) - } - } - err := s.Err() - if err != nil { - t.Fatalf("unexpected error; got %s", err) - } -} - -func TestSegmentLongTokensDontDouble(t *testing.T) { - // Read the data. - text := bytes.Repeat([]byte("abcdefghijklmnop"), 257) - buf := strings.NewReader(string(text)) - s := NewSegmenter(&slowReader{1, buf}) - // change to line segmenter for testing - s.SetSegmenter(wrapSplitFuncAsSegmentFuncForTesting(bufio.ScanLines)) - s.MaxTokenSize(6144) - for s.Segment() { - line := s.Bytes() - if !bytes.Equal(text, line) { - t.Errorf("expected %s, got %s", text, line) - } - } - err := s.Err() - if err != nil { - t.Fatalf("unexpected error; got %s", err) - } -} diff --git a/vendor/github.com/blevesearch/segment/segment_words_test.go b/vendor/github.com/blevesearch/segment/segment_words_test.go deleted file mode 100644 index 2c3c0fb8..00000000 --- a/vendor/github.com/blevesearch/segment/segment_words_test.go +++ /dev/null @@ -1,445 +0,0 @@ -// Copyright (c) 2014 Couchbase, Inc. -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software distributed under the -// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -// either express or implied. See the License for the specific language governing permissions -// and limitations under the License. - -package segment - -import ( - "bufio" - "bytes" - "reflect" - "strings" - "testing" -) - -func TestAdhocSegmentsWithType(t *testing.T) { - - tests := []struct { - input []byte - output [][]byte - outputStrings []string - outputTypes []int - }{ - { - input: []byte("Now is the.\n End."), - output: [][]byte{ - []byte("Now"), - []byte(" "), - []byte(" "), - []byte("is"), - []byte(" "), - []byte("the"), - []byte("."), - []byte("\n"), - []byte(" "), - []byte("End"), - []byte("."), - }, - outputStrings: []string{ - "Now", - " ", - " ", - "is", - " ", - "the", - ".", - "\n", - " ", - "End", - ".", - }, - outputTypes: []int{ - Letter, - None, - None, - Letter, - None, - Letter, - None, - None, - None, - Letter, - None, - }, - }, - { - input: []byte("3.5"), - output: [][]byte{ - []byte("3.5"), - }, - outputStrings: []string{ - "3.5", - }, - outputTypes: []int{ - Number, - }, - }, - { - input: []byte("cat3.5"), - output: [][]byte{ - []byte("cat3.5"), - }, - outputStrings: []string{ - "cat3.5", - }, - outputTypes: []int{ - Letter, - }, - }, - { - input: []byte("c"), - output: [][]byte{ - []byte("c"), - }, - outputStrings: []string{ - "c", - }, - outputTypes: []int{ - Letter, - }, - }, - { - input: []byte("こんにちは世界"), - output: [][]byte{ - []byte("こ"), - []byte("ん"), - []byte("に"), - []byte("ち"), - []byte("は"), - []byte("世"), - []byte("界"), - }, - outputStrings: []string{ - "こ", - "ん", - "に", - "ち", - "は", - "世", - "界", - }, - outputTypes: []int{ - Ideo, - Ideo, - Ideo, - Ideo, - Ideo, - Ideo, - Ideo, - }, - }, - { - input: []byte("你好世界"), - output: [][]byte{ - []byte("你"), - []byte("好"), - []byte("世"), - []byte("界"), - }, - outputStrings: []string{ - "你", - "好", - "世", - "界", - }, - outputTypes: []int{ - Ideo, - Ideo, - Ideo, - Ideo, - }, - }, - { - input: []byte("サッカ"), - output: [][]byte{ - []byte("サッカ"), - }, - outputStrings: []string{ - "サッカ", - }, - outputTypes: []int{ - Ideo, - }, - }, - // test for wb7b/wb7c - { - input: []byte(`א"א`), - output: [][]byte{ - []byte(`א"א`), - }, - outputStrings: []string{ - `א"א`, - }, - outputTypes: []int{ - Letter, - }, - }, - } - - for _, test := range tests { - rv := make([][]byte, 0) - rvstrings := make([]string, 0) - rvtypes := make([]int, 0) - segmenter := NewWordSegmenter(bytes.NewReader(test.input)) - // Set the split function for the scanning operation. - for segmenter.Segment() { - rv = append(rv, segmenter.Bytes()) - rvstrings = append(rvstrings, segmenter.Text()) - rvtypes = append(rvtypes, segmenter.Type()) - } - if err := segmenter.Err(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(rv, test.output) { - t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.output, rv, test.input) - } - if !reflect.DeepEqual(rvstrings, test.outputStrings) { - t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputStrings, rvstrings, test.input) - } - if !reflect.DeepEqual(rvtypes, test.outputTypes) { - t.Fatalf("expeced:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputTypes, rvtypes, test.input) - } - } - - // run same tests again with direct - for _, test := range tests { - rv := make([][]byte, 0) - rvstrings := make([]string, 0) - rvtypes := make([]int, 0) - segmenter := NewWordSegmenterDirect(test.input) - // Set the split function for the scanning operation. - for segmenter.Segment() { - rv = append(rv, segmenter.Bytes()) - rvstrings = append(rvstrings, segmenter.Text()) - rvtypes = append(rvtypes, segmenter.Type()) - } - if err := segmenter.Err(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(rv, test.output) { - t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.output, rv, test.input) - } - if !reflect.DeepEqual(rvstrings, test.outputStrings) { - t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputStrings, rvstrings, test.input) - } - if !reflect.DeepEqual(rvtypes, test.outputTypes) { - t.Fatalf("expeced:\n%#v\ngot:\n%#v\nfor: '%s'", test.outputTypes, rvtypes, test.input) - } - } - -} - -func TestUnicodeSegments(t *testing.T) { - - for _, test := range unicodeWordTests { - rv := make([][]byte, 0) - scanner := bufio.NewScanner(bytes.NewReader(test.input)) - // Set the split function for the scanning operation. - scanner.Split(SplitWords) - for scanner.Scan() { - rv = append(rv, scanner.Bytes()) - } - if err := scanner.Err(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(rv, test.output) { - t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: '%s' comment: %s", test.output, rv, test.input, test.comment) - } - } -} - -func TestUnicodeSegmentsSlowReader(t *testing.T) { - - for i, test := range unicodeWordTests { - rv := make([][]byte, 0) - segmenter := NewWordSegmenter(&slowReader{1, bytes.NewReader(test.input)}) - for segmenter.Segment() { - rv = append(rv, segmenter.Bytes()) - } - if err := segmenter.Err(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(rv, test.output) { - t.Fatalf("expected:\n%#v\ngot:\n%#v\nfor: %d '%s' comment: %s", test.output, rv, i, test.input, test.comment) - } - } -} - -func TestWordSegmentLongInputSlowReader(t *testing.T) { - // Read the data. - text := bytes.Repeat([]byte("abcdefghijklmnop"), 26) - buf := strings.NewReader(string(text) + " cat") - s := NewSegmenter(&slowReader{1, buf}) - s.MaxTokenSize(6144) - for s.Segment() { - } - err := s.Err() - if err != nil { - t.Fatalf("unexpected error; got '%s'", err) - } - finalWord := s.Text() - if s.Text() != "cat" { - t.Errorf("expected 'cat' got '%s'", finalWord) - } -} - -func BenchmarkSplitWords(b *testing.B) { - for i := 0; i < b.N; i++ { - vals := make([][]byte, 0) - scanner := bufio.NewScanner(bytes.NewReader(bleveWikiArticle)) - scanner.Split(SplitWords) - for scanner.Scan() { - vals = append(vals, scanner.Bytes()) - } - if err := scanner.Err(); err != nil { - b.Fatal(err) - } - if len(vals) != 3465 { - b.Fatalf("expected 3465 tokens, got %d", len(vals)) - } - } - -} - -func BenchmarkWordSegmenter(b *testing.B) { - - for i := 0; i < b.N; i++ { - vals := make([][]byte, 0) - types := make([]int, 0) - segmenter := NewWordSegmenter(bytes.NewReader(bleveWikiArticle)) - for segmenter.Segment() { - vals = append(vals, segmenter.Bytes()) - types = append(types, segmenter.Type()) - } - if err := segmenter.Err(); err != nil { - b.Fatal(err) - } - if vals == nil { - b.Fatalf("expected non-nil vals") - } - if types == nil { - b.Fatalf("expected non-nil types") - } - } -} - -func BenchmarkWordSegmenterDirect(b *testing.B) { - - for i := 0; i < b.N; i++ { - vals := make([][]byte, 0) - types := make([]int, 0) - segmenter := NewWordSegmenterDirect(bleveWikiArticle) - for segmenter.Segment() { - vals = append(vals, segmenter.Bytes()) - types = append(types, segmenter.Type()) - } - if err := segmenter.Err(); err != nil { - b.Fatal(err) - } - if vals == nil { - b.Fatalf("expected non-nil vals") - } - if types == nil { - b.Fatalf("expected non-nil types") - } - } -} - -func BenchmarkDirect(b *testing.B) { - - for i := 0; i < b.N; i++ { - vals := make([][]byte, 0, 10000) - types := make([]int, 0, 10000) - vals, types, _, err := SegmentWordsDirect(bleveWikiArticle, vals, types) - if err != nil { - b.Fatal(err) - } - if vals == nil { - b.Fatalf("expected non-nil vals") - } - if types == nil { - b.Fatalf("expected non-nil types") - } - } -} - -var bleveWikiArticle = []byte(`Boiling liquid expanding vapor explosion -From Wikipedia, the free encyclopedia -See also: Boiler explosion and Steam explosion - -Flames subsequent to a flammable liquid BLEVE from a tanker. BLEVEs do not necessarily involve fire. - -This article's tone or style may not reflect the encyclopedic tone used on Wikipedia. See Wikipedia's guide to writing better articles for suggestions. (July 2013) -A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.[1] -Contents [hide] -1 Mechanism -1.1 Water example -1.2 BLEVEs without chemical reactions -2 Fires -3 Incidents -4 Safety measures -5 See also -6 References -7 External links -Mechanism[edit] - -This section needs additional citations for verification. Please help improve this article by adding citations to reliable sources. Unsourced material may be challenged and removed. (July 2013) -There are three characteristics of liquids which are relevant to the discussion of a BLEVE: -If a liquid in a sealed container is boiled, the pressure inside the container increases. As the liquid changes to a gas it expands - this expansion in a vented container would cause the gas and liquid to take up more space. In a sealed container the gas and liquid are not able to take up more space and so the pressure rises. Pressurized vessels containing liquids can reach an equilibrium where the liquid stops boiling and the pressure stops rising. This occurs when no more heat is being added to the system (either because it has reached ambient temperature or has had a heat source removed). -The boiling temperature of a liquid is dependent on pressure - high pressures will yield high boiling temperatures, and low pressures will yield low boiling temperatures. A common simple experiment is to place a cup of water in a vacuum chamber, and then reduce the pressure in the chamber until the water boils. By reducing the pressure the water will boil even at room temperature. This works both ways - if the pressure is increased beyond normal atmospheric pressures, the boiling of hot water could be suppressed far beyond normal temperatures. The cooling system of a modern internal combustion engine is a real-world example. -When a liquid boils it turns into a gas. The resulting gas takes up far more space than the liquid did. -Typically, a BLEVE starts with a container of liquid which is held above its normal, atmospheric-pressure boiling temperature. Many substances normally stored as liquids, such as CO2, propane, and other similar industrial gases have boiling temperatures, at atmospheric pressure, far below room temperature. In the case of water, a BLEVE could occur if a pressurized chamber of water is heated far beyond the standard 100 °C (212 °F). That container, because the boiling water pressurizes it, is capable of holding liquid water at very high temperatures. -If the pressurized vessel, containing liquid at high temperature (which may be room temperature, depending on the substance) ruptures, the pressure which prevents the liquid from boiling is lost. If the rupture is catastrophic, where the vessel is immediately incapable of holding any pressure at all, then there suddenly exists a large mass of liquid which is at very high temperature and very low pressure. This causes the entire volume of liquid to instantaneously boil, which in turn causes an extremely rapid expansion. Depending on temperatures, pressures and the substance involved, that expansion may be so rapid that it can be classified as an explosion, fully capable of inflicting severe damage on its surroundings. -Water example[edit] -Imagine, for example, a tank of pressurized liquid water held at 204.4 °C (400 °F). This tank would normally be pressurized to 1.7 MPa (250 psi) above atmospheric ("gauge") pressure. If the tank containing the water were to rupture, there would for a slight moment exist a volume of liquid water which would be -at atmospheric pressure, and -204.4 °C (400 °F). -At atmospheric pressure the boiling point of water is 100 °C (212 °F) - liquid water at atmospheric pressure cannot exist at temperatures higher than 100 °C (212 °F). At that moment, the water would boil and turn to vapour explosively, and the 204.4 °C (400 °F) liquid water turned to gas would take up a lot more volume than it did as liquid, causing a vapour explosion. Such explosions can happen when the superheated water of a steam engine escapes through a crack in a boiler, causing a boiler explosion. -BLEVEs without chemical reactions[edit] -It is important to note that a BLEVE need not be a chemical explosion—nor does there need to be a fire—however if a flammable substance is subject to a BLEVE it may also be subject to intense heating, either from an external source of heat which may have caused the vessel to rupture in the first place or from an internal source of localized heating such as skin friction. This heating can cause a flammable substance to ignite, adding a secondary explosion caused by the primary BLEVE. While blast effects of any BLEVE can be devastating, a flammable substance such as propane can add significantly to the danger. -Bleve explosion.svg -While the term BLEVE is most often used to describe the results of a container of flammable liquid rupturing due to fire, a BLEVE can occur even with a non-flammable substance such as water,[2] liquid nitrogen,[3] liquid helium or other refrigerants or cryogens, and therefore is not usually considered a type of chemical explosion. -Fires[edit] -BLEVEs can be caused by an external fire near the storage vessel causing heating of the contents and pressure build-up. While tanks are often designed to withstand great pressure, constant heating can cause the metal to weaken and eventually fail. If the tank is being heated in an area where there is no liquid, it may rupture faster without the liquid to absorb the heat. Gas containers are usually equipped with relief valves that vent off excess pressure, but the tank can still fail if the pressure is not released quickly enough.[1] Relief valves are sized to release pressure fast enough to prevent the pressure from increasing beyond the strength of the vessel, but not so fast as to be the cause of an explosion. An appropriately sized relief valve will allow the liquid inside to boil slowly, maintaining a constant pressure in the vessel until all the liquid has boiled and the vessel empties. -If the substance involved is flammable, it is likely that the resulting cloud of the substance will ignite after the BLEVE has occurred, forming a fireball and possibly a fuel-air explosion, also termed a vapor cloud explosion (VCE). If the materials are toxic, a large area will be contaminated.[4] -Incidents[edit] -The term "BLEVE" was coined by three researchers at Factory Mutual, in the analysis of an accident there in 1957 involving a chemical reactor vessel.[5] -In August 1959 the Kansas City Fire Department suffered its largest ever loss of life in the line of duty, when a 25,000 gallon (95,000 litre) gas tank exploded during a fire on Southwest Boulevard killing five firefighters. This was the first time BLEVE was used to describe a burning fuel tank.[citation needed] -Later incidents included the Cheapside Street Whisky Bond Fire in Glasgow, Scotland in 1960; Feyzin, France in 1966; Crescent City, Illinois in 1970; Kingman, Arizona in 1973; a liquid nitrogen tank rupture[6] at Air Products and Chemicals and Mobay Chemical Company at New Martinsville, West Virginia on January 31, 1978 [1];Texas City, Texas in 1978; Murdock, Illinois in 1983; San Juan Ixhuatepec, Mexico City in 1984; and Toronto, Ontario in 2008. -Safety measures[edit] -[icon] This section requires expansion. (July 2013) -Some fire mitigation measures are listed under liquefied petroleum gas. -See also[edit] -Boiler explosion -Expansion ratio -Explosive boiling or phase explosion -Rapid phase transition -Viareggio train derailment -2008 Toronto explosions -Gas carriers -Los Alfaques Disaster -Lac-Mégantic derailment -References[edit] -^ Jump up to: a b Kletz, Trevor (March 1990). Critical Aspects of Safety and Loss Prevention. London: Butterworth–Heinemann. pp. 43–45. ISBN 0-408-04429-2. -Jump up ^ "Temperature Pressure Relief Valves on Water Heaters: test, inspect, replace, repair guide". Inspect-ny.com. Retrieved 2011-07-12. -Jump up ^ Liquid nitrogen BLEVE demo -Jump up ^ "Chemical Process Safety" (PDF). Retrieved 2011-07-12. -Jump up ^ David F. Peterson, BLEVE: Facts, Risk Factors, and Fallacies, Fire Engineering magazine (2002). -Jump up ^ "STATE EX REL. VAPOR CORP. v. NARICK". Supreme Court of Appeals of West Virginia. 1984-07-12. Retrieved 2014-03-16. -External links[edit] - Look up boiling liquid expanding vapor explosion in Wiktionary, the free dictionary. - Wikimedia Commons has media related to BLEVE. -BLEVE Demo on YouTube — video of a controlled BLEVE demo -huge explosions on YouTube — video of propane and isobutane BLEVEs from a train derailment at Murdock, Illinois (3 September 1983) -Propane BLEVE on YouTube — video of BLEVE from the Toronto propane depot fire -Moscow Ring Road Accident on YouTube - Dozens of LPG tank BLEVEs after a road accident in Moscow -Kingman, AZ BLEVE — An account of the 5 July 1973 explosion in Kingman, with photographs -Propane Tank Explosions — Description of circumstances required to cause a propane tank BLEVE. -Analysis of BLEVE Events at DOE Sites - Details physics and mathematics of BLEVEs. -HID - SAFETY REPORT ASSESSMENT GUIDE: Whisky Maturation Warehouses - The liquor is aged in wooden barrels that can suffer BLEVE. -Categories: ExplosivesFirefightingFireTypes of fireGas technologiesIndustrial fires and explosions`) diff --git a/vendor/github.com/blevesearch/segment/tables_test.go b/vendor/github.com/blevesearch/segment/tables_test.go deleted file mode 100644 index 7a39fd40..00000000 --- a/vendor/github.com/blevesearch/segment/tables_test.go +++ /dev/null @@ -1,11994 +0,0 @@ -// Generated by running -// maketesttables --url=http://www.unicode.org/Public/8.0.0/ucd/auxiliary/ -// DO NOT EDIT - -package segment - -var unicodeGraphemeTests = []struct { - input []byte - output [][]byte - comment string -}{ - { - input: []byte{0x20, 0x20}, - output: [][]byte{[]byte{0x20}, []byte{0x20}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xd}, - output: [][]byte{[]byte{0x20}, []byte{0xd}}, - comment: `÷ [0.2] SPACE (Other) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xa}, - output: [][]byte{[]byte{0x20}, []byte{0xa}}, - comment: `÷ [0.2] SPACE (Other) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0x1}, - output: [][]byte{[]byte{0x20}, []byte{0x1}}, - comment: `÷ [0.2] SPACE (Other) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x80}, - output: [][]byte{[]byte{0x20, 0xcc, 0x80}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0x20, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] SPACE (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0x20}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0x20}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0x20}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0x20}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0x20}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x20}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcd, 0xb8}, - output: [][]byte{[]byte{0x20}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] SPACE (Other) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0x20}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] SPACE (Other) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0x20, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x20}, - output: [][]byte{[]byte{0xd}, []byte{0x20}}, - comment: `÷ [0.2] (CR) ÷ [4.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xd}, - output: [][]byte{[]byte{0xd}, []byte{0xd}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xa}, - output: [][]byte{[]byte{0xd, 0xa}}, - comment: `÷ [0.2] (CR) × [3.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x1}, - output: [][]byte{[]byte{0xd}, []byte{0x1}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xd}, []byte{0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (CR) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xd}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (CR) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xd}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (CR) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xd}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (CR) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xd}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xd}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x20}, - output: [][]byte{[]byte{0xa}, []byte{0x20}}, - comment: `÷ [0.2] (LF) ÷ [4.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xd}, - output: [][]byte{[]byte{0xa}, []byte{0xd}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xa}, - output: [][]byte{[]byte{0xa}, []byte{0xa}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x1}, - output: [][]byte{[]byte{0xa}, []byte{0x1}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xa}, []byte{0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (LF) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xa}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (LF) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xa}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (LF) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xa}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (LF) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xa}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xa}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x20}, - output: [][]byte{[]byte{0x1}, []byte{0x20}}, - comment: `÷ [0.2] (Control) ÷ [4.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xd}, - output: [][]byte{[]byte{0x1}, []byte{0xd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xa}, - output: [][]byte{[]byte{0x1}, []byte{0xa}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x1}, - output: [][]byte{[]byte{0x1}, []byte{0x1}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x80}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0x1}, []byte{0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0x1}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0x1}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0x1}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0x1}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0x1}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x1}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcd, 0xb8}, - output: [][]byte{[]byte{0x1}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0x1}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0x1}, []byte{0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x20}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x20}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xd}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xa}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xa}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x1}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xcc, 0x80, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0x20}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0x20}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xd}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xd}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xa}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xa}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0x1}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0x1}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x80}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe0, 0xa4, 0x83, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe0, 0xa4, 0x83, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] DEVANAGARI SIGN VISARGA (SpacingMark) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0x20}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xd}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xa}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0x1}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [6.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x84, 0x80, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe1, 0x84, 0x80, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL CHOSEONG KIYEOK (L) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0x20}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xd}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xa}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0x1}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [7.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [7.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x85, 0xa0, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe1, 0x85, 0xa0, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL JUNGSEONG FILLER (V) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0x20}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xd}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xa}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0x1}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [8.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xe1, 0x86, 0xa8, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xe1, 0x86, 0xa8, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL JONGSEONG KIYEOK (T) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0x20}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xa}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0x1}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [7.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [7.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x80, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x80, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GA (LV) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0x20}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xa}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0x1}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [8.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xea, 0xb0, 0x81, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xea, 0xb0, 0x81, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] HANGUL SYLLABLE GAG (LVT) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x20}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x20}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xd}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xd}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x1}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x1}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0x20}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0x20}}, - comment: `÷ [0.2] (Other) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xd}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xd}}, - comment: `÷ [0.2] (Other) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xa}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xa}}, - comment: `÷ [0.2] (Other) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0x1}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0x1}}, - comment: `÷ [0.2] (Other) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x80}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (Other) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (Other) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (Other) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (Other) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (Other) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (Other) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (Other) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xcd, 0xb8}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (Other) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xcd, 0xb8, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xcd, 0xb8, 0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (Other) × [9.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0x20}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0x20}}, - comment: `÷ [0.2] (Control) ÷ [4.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0x20}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0x20}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xd}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xa}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xa}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0x1}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0x1}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x80}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.0] COMBINING GRAVE ACCENT (Extend) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (Control) ÷ [4.0] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xe0, 0xa4, 0x83}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88, 0xe0, 0xa4, 0x83}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) × [9.1] DEVANAGARI SIGN VISARGA (SpacingMark) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xe1, 0x84, 0x80}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xe1, 0x84, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL CHOSEONG KIYEOK (L) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xe1, 0x85, 0xa0}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xe1, 0x85, 0xa0}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JUNGSEONG FILLER (V) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xe1, 0x86, 0xa8}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xe1, 0x86, 0xa8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL JONGSEONG KIYEOK (T) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xea, 0xb0, 0x80}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x80}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GA (LV) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (Control) ÷ [4.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xea, 0xb0, 0x81}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xea, 0xb0, 0x81}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] HANGUL SYLLABLE GAG (LVT) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xcd, 0xb8}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xcd, 0xb8}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0xef, 0xbf, 0xbd, 0xcc, 0x88, 0xef, 0xbf, 0xbd}, - output: [][]byte{[]byte{0xef, 0xbf, 0xbd}, []byte{0xcc, 0x88}, []byte{0xef, 0xbf, 0xbd}}, - comment: `÷ [0.2] (Control) ÷ [4.0] COMBINING DIAERESIS (Extend) ÷ [5.0] (Control) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xf0, 0x9f, 0x87, 0xa6, 0x62}, - output: [][]byte{[]byte{0x61}, []byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x62}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN SMALL LETTER B (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER S (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER S (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER E (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xe2, 0x80, 0x8b, 0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba}, []byte{0xe2, 0x80, 0x8b}, []byte{0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) ÷ [5.0] ZERO WIDTH SPACE (Control) ÷ [4.0] REGIONAL INDICATOR SYMBOL LETTER S (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER E (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER B (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER C (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xe2, 0x80, 0x8d}, []byte{0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [9.0] ZERO WIDTH JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER B (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER C (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xe2, 0x80, 0x8d}, []byte{0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [8.1] REGIONAL INDICATOR SYMBOL LETTER B (Regional_Indicator) × [9.0] ZERO WIDTH JOINER (Extend) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER C (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xe2, 0x80, 0x8d, 0xd9, 0x86}, - output: [][]byte{[]byte{0x20, 0xe2, 0x80, 0x8d}, []byte{0xd9, 0x86}}, - comment: `÷ [0.2] SPACE (Other) × [9.0] ZERO WIDTH JOINER (Extend) ÷ [999.0] ARABIC LETTER NOON (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd9, 0x86, 0xe2, 0x80, 0x8d, 0x20}, - output: [][]byte{[]byte{0xd9, 0x86, 0xe2, 0x80, 0x8d}, []byte{0x20}}, - comment: `÷ [0.2] ARABIC LETTER NOON (Other) × [9.0] ZERO WIDTH JOINER (Extend) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, -} -var unicodeWordTests = []struct { - input []byte - output [][]byte - comment string -}{ - { - input: []byte{0x1, 0x1}, - output: [][]byte{[]byte{0x1}, []byte{0x1}}, - comment: `÷ [0.2] (Other) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xd}, - output: [][]byte{[]byte{0x1}, []byte{0xd}}, - comment: `÷ [0.2] (Other) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xa}, - output: [][]byte{[]byte{0x1}, []byte{0xa}}, - comment: `÷ [0.2] (Other) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xb}, - output: [][]byte{[]byte{0x1}, []byte{0xb}}, - comment: `÷ [0.2] (Other) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x1}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (Other) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x41}, - output: [][]byte{[]byte{0x1}, []byte{0x41}}, - comment: `÷ [0.2] (Other) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x3a}, - output: [][]byte{[]byte{0x1}, []byte{0x3a}}, - comment: `÷ [0.2] (Other) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x2c}, - output: [][]byte{[]byte{0x1}, []byte{0x2c}}, - comment: `÷ [0.2] (Other) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x2e}, - output: [][]byte{[]byte{0x1}, []byte{0x2e}}, - comment: `÷ [0.2] (Other) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x30}, - output: [][]byte{[]byte{0x1}, []byte{0x30}}, - comment: `÷ [0.2] (Other) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x5f}, - output: [][]byte{[]byte{0x1}, []byte{0x5f}}, - comment: `÷ [0.2] (Other) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x1}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xd7, 0x90}, - output: [][]byte{[]byte{0x1}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (Other) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x22}, - output: [][]byte{[]byte{0x1}, []byte{0x22}}, - comment: `÷ [0.2] (Other) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x27}, - output: [][]byte{[]byte{0x1}, []byte{0x27}}, - comment: `÷ [0.2] (Other) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xc2, 0xad}, - output: [][]byte{[]byte{0x1, 0xc2, 0xad}}, - comment: `÷ [0.2] (Other) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x80}, - output: [][]byte{[]byte{0x1, 0xcc, 0x80}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x1}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Other) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x61, 0x3a}, - output: [][]byte{[]byte{0x1}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (Other) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x61, 0x27}, - output: [][]byte{[]byte{0x1}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (Other) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x1}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Other) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x61, 0x2c}, - output: [][]byte{[]byte{0x1}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (Other) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x31, 0x3a}, - output: [][]byte{[]byte{0x1}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (Other) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x31, 0x27}, - output: [][]byte{[]byte{0x1}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (Other) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x31, 0x2c}, - output: [][]byte{[]byte{0x1}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (Other) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x1}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Other) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Other) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x1}, - output: [][]byte{[]byte{0xd}, []byte{0x1}}, - comment: `÷ [0.2] (CR) ÷ [3.1] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xd}, - output: [][]byte{[]byte{0xd}, []byte{0xd}}, - comment: `÷ [0.2] (CR) ÷ [3.1] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xa}, - output: [][]byte{[]byte{0xd, 0xa}}, - comment: `÷ [0.2] (CR) × [3.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xb}, - output: [][]byte{[]byte{0xd}, []byte{0xb}}, - comment: `÷ [0.2] (CR) ÷ [3.1] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xd}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (CR) ÷ [3.1] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x41}, - output: [][]byte{[]byte{0xd}, []byte{0x41}}, - comment: `÷ [0.2] (CR) ÷ [3.1] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x3a}, - output: [][]byte{[]byte{0xd}, []byte{0x3a}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0x2c}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x2e}, - output: [][]byte{[]byte{0xd}, []byte{0x2e}}, - comment: `÷ [0.2] (CR) ÷ [3.1] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x30}, - output: [][]byte{[]byte{0xd}, []byte{0x30}}, - comment: `÷ [0.2] (CR) ÷ [3.1] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x5f}, - output: [][]byte{[]byte{0xd}, []byte{0x5f}}, - comment: `÷ [0.2] (CR) ÷ [3.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xd}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (CR) ÷ [3.1] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xd7, 0x90}, - output: [][]byte{[]byte{0xd}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (CR) ÷ [3.1] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x22}, - output: [][]byte{[]byte{0xd}, []byte{0x22}}, - comment: `÷ [0.2] (CR) ÷ [3.1] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x27}, - output: [][]byte{[]byte{0xd}, []byte{0x27}}, - comment: `÷ [0.2] (CR) ÷ [3.1] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xc2, 0xad}, - output: [][]byte{[]byte{0xd}, []byte{0xc2, 0xad}}, - comment: `÷ [0.2] (CR) ÷ [3.1] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [3.1] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x61, 0x3a}, - output: [][]byte{[]byte{0xd}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (CR) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x61, 0x27}, - output: [][]byte{[]byte{0xd}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (CR) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x61, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (CR) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x31, 0x3a}, - output: [][]byte{[]byte{0xd}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (CR) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x31, 0x27}, - output: [][]byte{[]byte{0xd}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (CR) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x31, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (CR) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (CR) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x1}, - output: [][]byte{[]byte{0xa}, []byte{0x1}}, - comment: `÷ [0.2] (LF) ÷ [3.1] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xd}, - output: [][]byte{[]byte{0xa}, []byte{0xd}}, - comment: `÷ [0.2] (LF) ÷ [3.1] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xa}, - output: [][]byte{[]byte{0xa}, []byte{0xa}}, - comment: `÷ [0.2] (LF) ÷ [3.1] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xb}, - output: [][]byte{[]byte{0xa}, []byte{0xb}}, - comment: `÷ [0.2] (LF) ÷ [3.1] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xa}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (LF) ÷ [3.1] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x41}, - output: [][]byte{[]byte{0xa}, []byte{0x41}}, - comment: `÷ [0.2] (LF) ÷ [3.1] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x3a}, - output: [][]byte{[]byte{0xa}, []byte{0x3a}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0x2c}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x2e}, - output: [][]byte{[]byte{0xa}, []byte{0x2e}}, - comment: `÷ [0.2] (LF) ÷ [3.1] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x30}, - output: [][]byte{[]byte{0xa}, []byte{0x30}}, - comment: `÷ [0.2] (LF) ÷ [3.1] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x5f}, - output: [][]byte{[]byte{0xa}, []byte{0x5f}}, - comment: `÷ [0.2] (LF) ÷ [3.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xa}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (LF) ÷ [3.1] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xd7, 0x90}, - output: [][]byte{[]byte{0xa}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (LF) ÷ [3.1] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x22}, - output: [][]byte{[]byte{0xa}, []byte{0x22}}, - comment: `÷ [0.2] (LF) ÷ [3.1] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x27}, - output: [][]byte{[]byte{0xa}, []byte{0x27}}, - comment: `÷ [0.2] (LF) ÷ [3.1] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xc2, 0xad}, - output: [][]byte{[]byte{0xa}, []byte{0xc2, 0xad}}, - comment: `÷ [0.2] (LF) ÷ [3.1] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [3.1] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x61, 0x3a}, - output: [][]byte{[]byte{0xa}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (LF) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x61, 0x27}, - output: [][]byte{[]byte{0xa}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (LF) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x61, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (LF) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x31, 0x3a}, - output: [][]byte{[]byte{0xa}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (LF) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x31, 0x27}, - output: [][]byte{[]byte{0xa}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (LF) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x31, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (LF) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (LF) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x1}, - output: [][]byte{[]byte{0xb}, []byte{0x1}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xd}, - output: [][]byte{[]byte{0xb}, []byte{0xd}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xa}, - output: [][]byte{[]byte{0xb}, []byte{0xa}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xb}, - output: [][]byte{[]byte{0xb}, []byte{0xb}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xb}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x41}, - output: [][]byte{[]byte{0xb}, []byte{0x41}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x3a}, - output: [][]byte{[]byte{0xb}, []byte{0x3a}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x2c}, - output: [][]byte{[]byte{0xb}, []byte{0x2c}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x2e}, - output: [][]byte{[]byte{0xb}, []byte{0x2e}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x30}, - output: [][]byte{[]byte{0xb}, []byte{0x30}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x5f}, - output: [][]byte{[]byte{0xb}, []byte{0x5f}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xb}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xd7, 0x90}, - output: [][]byte{[]byte{0xb}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x22}, - output: [][]byte{[]byte{0xb}, []byte{0x22}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x27}, - output: [][]byte{[]byte{0xb}, []byte{0x27}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xc2, 0xad}, - output: [][]byte{[]byte{0xb}, []byte{0xc2, 0xad}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x80}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xb}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x61, 0x3a}, - output: [][]byte{[]byte{0xb}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x61, 0x27}, - output: [][]byte{[]byte{0xb}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xb}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x61, 0x2c}, - output: [][]byte{[]byte{0xb}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x31, 0x3a}, - output: [][]byte{[]byte{0xb}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x31, 0x27}, - output: [][]byte{[]byte{0xb}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x31, 0x2c}, - output: [][]byte{[]byte{0xb}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xb}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xb, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xb}, []byte{0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] (Newline) ÷ [3.1] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x1}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x1}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xd}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0xd}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xa}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0xa}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xb}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0xb}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [13.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x41}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x41}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x3a}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x3a}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x2c}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x2c}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x2e}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x2e}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x30}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x30}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x5f}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0x5f}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x5f}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xd7, 0x90}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x22}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x22}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x27}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x27}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xc2, 0xad}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xc2, 0xad}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x80}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x61, 0x3a}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x61, 0x27}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x61, 0x2c}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x31, 0x3a}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x31, 0x27}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x31, 0x2c}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe3, 0x80, 0xb1, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe3, 0x80, 0xb1, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] VERTICAL KANA REPEAT MARK (Katakana) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x1}, - output: [][]byte{[]byte{0x41}, []byte{0x1}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xd}, - output: [][]byte{[]byte{0x41}, []byte{0xd}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xa}, - output: [][]byte{[]byte{0x41}, []byte{0xa}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xb}, - output: [][]byte{[]byte{0x41}, []byte{0xb}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x41}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x41}, - output: [][]byte{[]byte{0x41, 0x41}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [5.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x3a}, - output: [][]byte{[]byte{0x41}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x2c}, - output: [][]byte{[]byte{0x41}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x2e}, - output: [][]byte{[]byte{0x41}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x30}, - output: [][]byte{[]byte{0x41, 0x30}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [9.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x5f}, - output: [][]byte{[]byte{0x41, 0x5f}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x5f}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x41}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xd7, 0x90}, - output: [][]byte{[]byte{0x41, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [5.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x22}, - output: [][]byte{[]byte{0x41}, []byte{0x22}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x27}, - output: [][]byte{[]byte{0x41}, []byte{0x27}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xc2, 0xad}, - output: [][]byte{[]byte{0x41, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x80}, - output: [][]byte{[]byte{0x41, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x41, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x61, 0x3a}, - output: [][]byte{[]byte{0x41, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x61, 0x27}, - output: [][]byte{[]byte{0x41, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x41, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x61, 0x2c}, - output: [][]byte{[]byte{0x41, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x31, 0x3a}, - output: [][]byte{[]byte{0x41, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x31, 0x27}, - output: [][]byte{[]byte{0x41, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x31, 0x2c}, - output: [][]byte{[]byte{0x41, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x41, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (ALetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x1}, - output: [][]byte{[]byte{0x3a}, []byte{0x1}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xd}, - output: [][]byte{[]byte{0x3a}, []byte{0xd}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xa}, - output: [][]byte{[]byte{0x3a}, []byte{0xa}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xb}, - output: [][]byte{[]byte{0x3a}, []byte{0xb}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x3a}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x41}, - output: [][]byte{[]byte{0x3a}, []byte{0x41}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x3a}, - output: [][]byte{[]byte{0x3a}, []byte{0x3a}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x2c}, - output: [][]byte{[]byte{0x3a}, []byte{0x2c}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x2e}, - output: [][]byte{[]byte{0x3a}, []byte{0x2e}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x30}, - output: [][]byte{[]byte{0x3a}, []byte{0x30}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x5f}, - output: [][]byte{[]byte{0x3a}, []byte{0x5f}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x3a}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xd7, 0x90}, - output: [][]byte{[]byte{0x3a}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x22}, - output: [][]byte{[]byte{0x3a}, []byte{0x22}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x27}, - output: [][]byte{[]byte{0x3a}, []byte{0x27}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xc2, 0xad}, - output: [][]byte{[]byte{0x3a, 0xc2, 0xad}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x80}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x80}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x3a}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x61, 0x3a}, - output: [][]byte{[]byte{0x3a}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x61, 0x27}, - output: [][]byte{[]byte{0x3a}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x3a}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x61, 0x2c}, - output: [][]byte{[]byte{0x3a}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x31, 0x3a}, - output: [][]byte{[]byte{0x3a}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x31, 0x27}, - output: [][]byte{[]byte{0x3a}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x31, 0x2c}, - output: [][]byte{[]byte{0x3a}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x3a}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x3a, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x1}, - output: [][]byte{[]byte{0x2c}, []byte{0x1}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xd}, - output: [][]byte{[]byte{0x2c}, []byte{0xd}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xa}, - output: [][]byte{[]byte{0x2c}, []byte{0xa}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xb}, - output: [][]byte{[]byte{0x2c}, []byte{0xb}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x2c}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x41}, - output: [][]byte{[]byte{0x2c}, []byte{0x41}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x3a}, - output: [][]byte{[]byte{0x2c}, []byte{0x3a}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x2c}, - output: [][]byte{[]byte{0x2c}, []byte{0x2c}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x2e}, - output: [][]byte{[]byte{0x2c}, []byte{0x2e}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x30}, - output: [][]byte{[]byte{0x2c}, []byte{0x30}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x5f}, - output: [][]byte{[]byte{0x2c}, []byte{0x5f}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x2c}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xd7, 0x90}, - output: [][]byte{[]byte{0x2c}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x22}, - output: [][]byte{[]byte{0x2c}, []byte{0x22}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x27}, - output: [][]byte{[]byte{0x2c}, []byte{0x27}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2c, 0xc2, 0xad}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x80}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2c}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x61, 0x3a}, - output: [][]byte{[]byte{0x2c}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x61, 0x27}, - output: [][]byte{[]byte{0x2c}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2c}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x61, 0x2c}, - output: [][]byte{[]byte{0x2c}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x31, 0x3a}, - output: [][]byte{[]byte{0x2c}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x31, 0x27}, - output: [][]byte{[]byte{0x2c}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x31, 0x2c}, - output: [][]byte{[]byte{0x2c}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2c}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x1}, - output: [][]byte{[]byte{0x2e}, []byte{0x1}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xd}, - output: [][]byte{[]byte{0x2e}, []byte{0xd}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xa}, - output: [][]byte{[]byte{0x2e}, []byte{0xa}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xb}, - output: [][]byte{[]byte{0x2e}, []byte{0xb}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x2e}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x41}, - output: [][]byte{[]byte{0x2e}, []byte{0x41}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x3a}, - output: [][]byte{[]byte{0x2e}, []byte{0x3a}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x2c}, - output: [][]byte{[]byte{0x2e}, []byte{0x2c}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x2e}, - output: [][]byte{[]byte{0x2e}, []byte{0x2e}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x30}, - output: [][]byte{[]byte{0x2e}, []byte{0x30}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x5f}, - output: [][]byte{[]byte{0x2e}, []byte{0x5f}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x2e}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xd7, 0x90}, - output: [][]byte{[]byte{0x2e}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x22}, - output: [][]byte{[]byte{0x2e}, []byte{0x22}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x27}, - output: [][]byte{[]byte{0x2e}, []byte{0x27}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2e, 0xc2, 0xad}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x80}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2e}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x61, 0x3a}, - output: [][]byte{[]byte{0x2e}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x61, 0x27}, - output: [][]byte{[]byte{0x2e}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2e}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x61, 0x2c}, - output: [][]byte{[]byte{0x2e}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x31, 0x3a}, - output: [][]byte{[]byte{0x2e}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x31, 0x27}, - output: [][]byte{[]byte{0x2e}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x31, 0x2c}, - output: [][]byte{[]byte{0x2e}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2e}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] FULL STOP (MidNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x1}, - output: [][]byte{[]byte{0x30}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xd}, - output: [][]byte{[]byte{0x30}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xa}, - output: [][]byte{[]byte{0x30}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xb}, - output: [][]byte{[]byte{0x30}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x30}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x41}, - output: [][]byte{[]byte{0x30, 0x41}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [10.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [10.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x3a}, - output: [][]byte{[]byte{0x30}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x2c}, - output: [][]byte{[]byte{0x30}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x2e}, - output: [][]byte{[]byte{0x30}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x30}, - output: [][]byte{[]byte{0x30, 0x30}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [8.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [8.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x5f}, - output: [][]byte{[]byte{0x30, 0x5f}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x5f}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x30}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xd7, 0x90}, - output: [][]byte{[]byte{0x30, 0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [10.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [10.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x22}, - output: [][]byte{[]byte{0x30}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x27}, - output: [][]byte{[]byte{0x30}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xc2, 0xad}, - output: [][]byte{[]byte{0x30, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x80}, - output: [][]byte{[]byte{0x30, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x30, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [10.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [10.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x61, 0x3a}, - output: [][]byte{[]byte{0x30, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x61, 0x27}, - output: [][]byte{[]byte{0x30, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x30, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x61, 0x2c}, - output: [][]byte{[]byte{0x30, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x31, 0x3a}, - output: [][]byte{[]byte{0x30, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x31, 0x27}, - output: [][]byte{[]byte{0x30, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x31, 0x2c}, - output: [][]byte{[]byte{0x30, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x30, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [4.0] COMBINING DIAERESIS (Extend_FE) × [8.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x1}, - output: [][]byte{[]byte{0x5f}, []byte{0x1}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xd}, - output: [][]byte{[]byte{0x5f}, []byte{0xd}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xa}, - output: [][]byte{[]byte{0x5f}, []byte{0xa}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xb}, - output: [][]byte{[]byte{0x5f}, []byte{0xb}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x5f, 0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x41}, - output: [][]byte{[]byte{0x5f, 0x41}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x3a}, - output: [][]byte{[]byte{0x5f}, []byte{0x3a}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x2c}, - output: [][]byte{[]byte{0x5f}, []byte{0x2c}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x2e}, - output: [][]byte{[]byte{0x5f}, []byte{0x2e}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x30}, - output: [][]byte{[]byte{0x5f, 0x30}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x5f}, - output: [][]byte{[]byte{0x5f, 0x5f}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x5f}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x5f}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xd7, 0x90}, - output: [][]byte{[]byte{0x5f, 0xd7, 0x90}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x22}, - output: [][]byte{[]byte{0x5f}, []byte{0x22}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x27}, - output: [][]byte{[]byte{0x5f}, []byte{0x27}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xc2, 0xad}, - output: [][]byte{[]byte{0x5f, 0xc2, 0xad}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x80}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x80}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x5f, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x61, 0x3a}, - output: [][]byte{[]byte{0x5f, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x61, 0x27}, - output: [][]byte{[]byte{0x5f, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x5f, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x61, 0x2c}, - output: [][]byte{[]byte{0x5f, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x31, 0x3a}, - output: [][]byte{[]byte{0x5f, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x31, 0x27}, - output: [][]byte{[]byte{0x5f, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x31, 0x2c}, - output: [][]byte{[]byte{0x5f, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x5f, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x5f, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x5f, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LOW LINE (ExtendNumLet) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x1}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x1}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xd}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xd}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xb}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xb}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x41}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x41}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x3a}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x3a}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x2c}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x2c}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x2e}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x2e}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x30}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x30}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x5f}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x5f}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.3] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xd7, 0x90}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x22}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x22}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x27}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x27}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xc2, 0xad}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xc2, 0xad}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x61, 0x3a}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x61, 0x27}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x61, 0x2c}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x31, 0x3a}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x31, 0x27}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x31, 0x2c}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x1}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0x1}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xd}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0xd}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xa}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0xa}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xb}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0xb}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x41}, - output: [][]byte{[]byte{0xd7, 0x90, 0x41}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [5.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x3a}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0x3a}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x2c}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0x2c}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x2e}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0x2e}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x30}, - output: [][]byte{[]byte{0xd7, 0x90, 0x30}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [9.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x5f}, - output: [][]byte{[]byte{0xd7, 0x90, 0x5f}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x5f}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xd7, 0x90}, - output: [][]byte{[]byte{0xd7, 0x90, 0xd7, 0x90}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [5.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x22}, - output: [][]byte{[]byte{0xd7, 0x90}, []byte{0x22}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x27}, - output: [][]byte{[]byte{0xd7, 0x90, 0x27}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [7.1] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x27}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.1] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xc2, 0xad}, - output: [][]byte{[]byte{0xd7, 0x90, 0xc2, 0xad}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x80}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd7, 0x90, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x61, 0x3a}, - output: [][]byte{[]byte{0xd7, 0x90, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x61, 0x27}, - output: [][]byte{[]byte{0xd7, 0x90, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd7, 0x90, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x61, 0x2c}, - output: [][]byte{[]byte{0xd7, 0x90, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x31, 0x3a}, - output: [][]byte{[]byte{0xd7, 0x90, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x31, 0x27}, - output: [][]byte{[]byte{0xd7, 0x90, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x31, 0x2c}, - output: [][]byte{[]byte{0xd7, 0x90, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd7, 0x90, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xd7, 0x90, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x1}, - output: [][]byte{[]byte{0x22}, []byte{0x1}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xd}, - output: [][]byte{[]byte{0x22}, []byte{0xd}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xa}, - output: [][]byte{[]byte{0x22}, []byte{0xa}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xb}, - output: [][]byte{[]byte{0x22}, []byte{0xb}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x22}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x41}, - output: [][]byte{[]byte{0x22}, []byte{0x41}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x3a}, - output: [][]byte{[]byte{0x22}, []byte{0x3a}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x2c}, - output: [][]byte{[]byte{0x22}, []byte{0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x2e}, - output: [][]byte{[]byte{0x22}, []byte{0x2e}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x30}, - output: [][]byte{[]byte{0x22}, []byte{0x30}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x5f}, - output: [][]byte{[]byte{0x22}, []byte{0x5f}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x22}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xd7, 0x90}, - output: [][]byte{[]byte{0x22}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x22}, - output: [][]byte{[]byte{0x22}, []byte{0x22}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x27}, - output: [][]byte{[]byte{0x22}, []byte{0x27}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xc2, 0xad}, - output: [][]byte{[]byte{0x22, 0xc2, 0xad}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x80}, - output: [][]byte{[]byte{0x22, 0xcc, 0x80}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x22}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x61, 0x3a}, - output: [][]byte{[]byte{0x22}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x61, 0x27}, - output: [][]byte{[]byte{0x22}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x22}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x61, 0x2c}, - output: [][]byte{[]byte{0x22}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x31, 0x3a}, - output: [][]byte{[]byte{0x22}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x31, 0x27}, - output: [][]byte{[]byte{0x22}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x31, 0x2c}, - output: [][]byte{[]byte{0x22}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x22}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] QUOTATION MARK (Double_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x1}, - output: [][]byte{[]byte{0x27}, []byte{0x1}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xd}, - output: [][]byte{[]byte{0x27}, []byte{0xd}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xa}, - output: [][]byte{[]byte{0x27}, []byte{0xa}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xb}, - output: [][]byte{[]byte{0x27}, []byte{0xb}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x27}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x41}, - output: [][]byte{[]byte{0x27}, []byte{0x41}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x3a}, - output: [][]byte{[]byte{0x27}, []byte{0x3a}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x2c}, - output: [][]byte{[]byte{0x27}, []byte{0x2c}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x2e}, - output: [][]byte{[]byte{0x27}, []byte{0x2e}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x30}, - output: [][]byte{[]byte{0x27}, []byte{0x30}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x5f}, - output: [][]byte{[]byte{0x27}, []byte{0x5f}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x27}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xd7, 0x90}, - output: [][]byte{[]byte{0x27}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x22}, - output: [][]byte{[]byte{0x27}, []byte{0x22}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x27}, - output: [][]byte{[]byte{0x27}, []byte{0x27}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xc2, 0xad}, - output: [][]byte{[]byte{0x27, 0xc2, 0xad}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x80}, - output: [][]byte{[]byte{0x27, 0xcc, 0x80}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x27}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x61, 0x3a}, - output: [][]byte{[]byte{0x27}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x61, 0x27}, - output: [][]byte{[]byte{0x27}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x27}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x61, 0x2c}, - output: [][]byte{[]byte{0x27}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x31, 0x3a}, - output: [][]byte{[]byte{0x27}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x31, 0x27}, - output: [][]byte{[]byte{0x27}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x31, 0x2c}, - output: [][]byte{[]byte{0x27}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x27}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x27, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x1}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x1}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xd}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0xd}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xa}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0xa}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xb}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0xb}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x41}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x41}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x3a}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x3a}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x2e}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x2e}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x30}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x30}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x5f}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x5f}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xd7, 0x90}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x22}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x22}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x27}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x27}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc2, 0xad, 0xc2, 0xad}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x80}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x61, 0x3a}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x61, 0x27}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x61, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x31, 0x3a}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x31, 0x27}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x31, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xc2, 0xad}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x1}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xd}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xa}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xa}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xb}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xb}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x41}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x41}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x3a}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x3a}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x2e}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x2e}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x30}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x30}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x5f}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x5f}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xd7, 0x90}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x22}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x22}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x27}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x27}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xc2, 0xad}, - output: [][]byte{[]byte{0xcc, 0x80, 0xc2, 0xad}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x61, 0x3a}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x61, 0x27}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x61, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x31, 0x3a}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x31, 0x27}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x31, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x1}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xd}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xa}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xb}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x41}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x3a}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x2c}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x2e}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x30}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [9.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x5f}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [13.1] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x22}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x27}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x31, 0x27}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [9.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x41}, - output: [][]byte{[]byte{0x61, 0x3a, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [7.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x61, 0x3a, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0x3a, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [7.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0x3a, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x3a, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [7.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x3a, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0x3a, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0x3a, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0x3a, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0x3a, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x3a, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x3a, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0x3a, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0x3a, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x41}, - output: [][]byte{[]byte{0x61, 0x27, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x61, 0x27, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0x27, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0x27, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0x27, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0x27, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0x27, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0x27, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0x27, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0x27, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x27}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x41}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [7.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x41}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x61, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x61}, []byte{0x2c, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x30}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x31, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x3a, 0xcc, 0x88}, []byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x30}, - output: [][]byte{[]byte{0x31, 0x27, 0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x31, 0x27, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x27}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x27, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31, 0x27, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31, 0x27, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x31, 0x27}, - output: [][]byte{[]byte{0x31, 0x27, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x31, 0x27, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31, 0x27, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31, 0x27, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31, 0x27, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x27, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31, 0x27, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x30}, - output: [][]byte{[]byte{0x31, 0x2c, 0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x31, 0x2c, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2c, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31, 0x2c, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31, 0x2c, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x31, 0x27}, - output: [][]byte{[]byte{0x31, 0x2c, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x31, 0x2c, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31, 0x2c, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31, 0x2c, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31, 0x2c, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31, 0x2c, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] COMMA (MidNum) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xd}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xa}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xb}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xb}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [3.2] (Newline) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xe3, 0x80, 0xb1}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xe3, 0x80, 0xb1}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] VERTICAL KANA REPEAT MARK (Katakana) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN CAPITAL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x2e}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] FULL STOP (MidNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x30}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x5f}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x5f}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LOW LINE (ExtendNumLet) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xf0, 0x9f, 0x87, 0xa6}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xf0, 0x9f, 0x87, 0xa6}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xd7, 0x90}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0xd7, 0x90}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x22}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] QUOTATION MARK (Double_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x61, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x3a}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x61}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x27}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x61}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x27, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x61}, []byte{0x27, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x61, 0x2c}, - output: [][]byte{[]byte{0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x61}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x3a}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x3a}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31, 0x27}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x27}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x27}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x2c}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x2c}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31, 0x2e, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0x31, 0x2e, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x31}, []byte{0x2e, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [4.0] COMBINING DIAERESIS (Extend_FE) × [11.0] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x63, 0x61, 0x6e, 0x27, 0x74}, - output: [][]byte{[]byte{0x63, 0x61, 0x6e, 0x27, 0x74}}, - comment: `÷ [0.2] LATIN SMALL LETTER C (ALetter) × [5.0] LATIN SMALL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER N (ALetter) × [6.0] APOSTROPHE (Single_Quote) × [7.0] LATIN SMALL LETTER T (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x63, 0x61, 0x6e, 0xe2, 0x80, 0x99, 0x74}, - output: [][]byte{[]byte{0x63, 0x61, 0x6e, 0xe2, 0x80, 0x99, 0x74}}, - comment: `÷ [0.2] LATIN SMALL LETTER C (ALetter) × [5.0] LATIN SMALL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER N (ALetter) × [6.0] RIGHT SINGLE QUOTATION MARK (MidNumLet) × [7.0] LATIN SMALL LETTER T (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x62, 0xc2, 0xad, 0x62, 0x79}, - output: [][]byte{[]byte{0x61, 0x62, 0xc2, 0xad, 0x62, 0x79}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [5.0] LATIN SMALL LETTER B (ALetter) × [4.0] SOFT HYPHEN (Format_FE) × [5.0] LATIN SMALL LETTER B (ALetter) × [5.0] LATIN SMALL LETTER Y (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x24, 0x2d, 0x33, 0x34, 0x2c, 0x35, 0x36, 0x37, 0x2e, 0x31, 0x34, 0x25, 0x62}, - output: [][]byte{[]byte{0x61}, []byte{0x24}, []byte{0x2d}, []byte{0x33, 0x34, 0x2c, 0x35, 0x36, 0x37, 0x2e, 0x31, 0x34}, []byte{0x25}, []byte{0x62}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] DOLLAR SIGN (Other) ÷ [999.0] HYPHEN-MINUS (Other) ÷ [999.0] DIGIT THREE (Numeric) × [8.0] DIGIT FOUR (Numeric) × [12.0] COMMA (MidNum) × [11.0] DIGIT FIVE (Numeric) × [8.0] DIGIT SIX (Numeric) × [8.0] DIGIT SEVEN (Numeric) × [12.0] FULL STOP (MidNumLet) × [11.0] DIGIT ONE (Numeric) × [8.0] DIGIT FOUR (Numeric) ÷ [999.0] PERCENT SIGN (Other) ÷ [999.0] LATIN SMALL LETTER B (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x33, 0x61}, - output: [][]byte{[]byte{0x33, 0x61}}, - comment: `÷ [0.2] DIGIT THREE (Numeric) × [10.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x63, 0x2e, 0x64}, - output: [][]byte{[]byte{0x63, 0x2e, 0x64}}, - comment: `÷ [0.2] LATIN SMALL LETTER C (ALetter) × [6.0] FULL STOP (MidNumLet) × [7.0] LATIN SMALL LETTER D (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x43, 0x2e, 0x64}, - output: [][]byte{[]byte{0x43, 0x2e, 0x64}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER C (ALetter) × [6.0] FULL STOP (MidNumLet) × [7.0] LATIN SMALL LETTER D (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x63, 0x2e, 0x44}, - output: [][]byte{[]byte{0x63, 0x2e, 0x44}}, - comment: `÷ [0.2] LATIN SMALL LETTER C (ALetter) × [6.0] FULL STOP (MidNumLet) × [7.0] LATIN CAPITAL LETTER D (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x43, 0x2e, 0x44}, - output: [][]byte{[]byte{0x43, 0x2e, 0x44}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER C (ALetter) × [6.0] FULL STOP (MidNumLet) × [7.0] LATIN CAPITAL LETTER D (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x6e, 0xe2, 0x81, 0xa0, 0x27, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x63, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x6e, 0xe2, 0x81, 0xa0, 0x27, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER C (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER N (ALetter) × [4.0] WORD JOINER (Format_FE) × [6.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER T (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x6e, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x63, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x6e, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER C (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER N (ALetter) × [4.0] WORD JOINER (Format_FE) × [6.0] RIGHT SINGLE QUOTATION MARK (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER T (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x62, 0xe2, 0x81, 0xa0, 0xc2, 0xad, 0xe2, 0x81, 0xa0, 0x62, 0xe2, 0x81, 0xa0, 0x79, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x61, 0xe2, 0x81, 0xa0, 0x62, 0xe2, 0x81, 0xa0, 0xc2, 0xad, 0xe2, 0x81, 0xa0, 0x62, 0xe2, 0x81, 0xa0, 0x79, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER B (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] SOFT HYPHEN (Format_FE) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER B (ALetter) × [4.0] WORD JOINER (Format_FE) × [5.0] LATIN SMALL LETTER Y (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x24, 0xe2, 0x81, 0xa0, 0x2d, 0xe2, 0x81, 0xa0, 0x33, 0xe2, 0x81, 0xa0, 0x34, 0xe2, 0x81, 0xa0, 0x2c, 0xe2, 0x81, 0xa0, 0x35, 0xe2, 0x81, 0xa0, 0x36, 0xe2, 0x81, 0xa0, 0x37, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x31, 0xe2, 0x81, 0xa0, 0x34, 0xe2, 0x81, 0xa0, 0x25, 0xe2, 0x81, 0xa0, 0x62, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x61, 0xe2, 0x81, 0xa0}, []byte{0x24, 0xe2, 0x81, 0xa0}, []byte{0x2d, 0xe2, 0x81, 0xa0}, []byte{0x33, 0xe2, 0x81, 0xa0, 0x34, 0xe2, 0x81, 0xa0, 0x2c, 0xe2, 0x81, 0xa0, 0x35, 0xe2, 0x81, 0xa0, 0x36, 0xe2, 0x81, 0xa0, 0x37, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x31, 0xe2, 0x81, 0xa0, 0x34, 0xe2, 0x81, 0xa0}, []byte{0x25, 0xe2, 0x81, 0xa0}, []byte{0x62, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DOLLAR SIGN (Other) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] HYPHEN-MINUS (Other) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] DIGIT THREE (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT FOUR (Numeric) × [4.0] WORD JOINER (Format_FE) × [12.0] COMMA (MidNum) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT FIVE (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT SIX (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT SEVEN (Numeric) × [4.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [11.0] DIGIT ONE (Numeric) × [4.0] WORD JOINER (Format_FE) × [8.0] DIGIT FOUR (Numeric) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] PERCENT SIGN (Other) × [4.0] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER B (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x33, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x33, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] DIGIT THREE (Numeric) × [4.0] WORD JOINER (Format_FE) × [10.0] LATIN SMALL LETTER A (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER C (ALetter) × [4.0] WORD JOINER (Format_FE) × [6.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER D (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN CAPITAL LETTER C (ALetter) × [4.0] WORD JOINER (Format_FE) × [6.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN SMALL LETTER D (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN SMALL LETTER C (ALetter) × [4.0] WORD JOINER (Format_FE) × [6.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER D (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0}, []byte{0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) ÷ [999.0] LATIN CAPITAL LETTER C (ALetter) × [4.0] WORD JOINER (Format_FE) × [6.0] FULL STOP (MidNumLet) × [4.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER D (ALetter) × [4.0] WORD JOINER (Format_FE) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xf0, 0x9f, 0x87, 0xa6, 0x62}, - output: [][]byte{[]byte{0x61}, []byte{0xf0, 0x9f, 0x87, 0xa6}, []byte{0x62}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) ÷ [999.0] LATIN SMALL LETTER B (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER S (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER S (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER E (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba, 0xe2, 0x80, 0x8b, 0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xb7, 0xf0, 0x9f, 0x87, 0xba}, []byte{0xe2, 0x80, 0x8b}, []byte{0xf0, 0x9f, 0x87, 0xb8, 0xf0, 0x9f, 0x87, 0xaa}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER R (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER U (Regional_Indicator) ÷ [999.0] ZERO WIDTH SPACE (Other) ÷ [999.0] REGIONAL INDICATOR SYMBOL LETTER S (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER E (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xd7, 0x90, 0x22, 0xd7, 0x90}, - output: [][]byte{[]byte{0xd7, 0x90, 0x22, 0xd7, 0x90}}, - comment: `÷ [0.2] HEBREW LETTER ALEF (Hebrew_Letter) × [7.2] QUOTATION MARK (Double_Quote) × [7.3] HEBREW LETTER ALEF (Hebrew_Letter) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER B (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER C (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [4.0] ZERO WIDTH JOINER (Extend_FE) × [13.3] REGIONAL INDICATOR SYMBOL LETTER B (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER C (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Regional_Indicator) × [13.3] REGIONAL INDICATOR SYMBOL LETTER B (Regional_Indicator) × [4.0] ZERO WIDTH JOINER (Extend_FE) × [13.3] REGIONAL INDICATOR SYMBOL LETTER C (Regional_Indicator) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xe2, 0x80, 0x8d, 0xd9, 0x86}, - output: [][]byte{[]byte{0x20, 0xe2, 0x80, 0x8d}, []byte{0xd9, 0x86}}, - comment: `÷ [0.2] SPACE (Other) × [4.0] ZERO WIDTH JOINER (Extend_FE) ÷ [999.0] ARABIC LETTER NOON (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0xd9, 0x86, 0xe2, 0x80, 0x8d, 0x20}, - output: [][]byte{[]byte{0xd9, 0x86, 0xe2, 0x80, 0x8d}, []byte{0x20}}, - comment: `÷ [0.2] ARABIC LETTER NOON (ALetter) × [4.0] ZERO WIDTH JOINER (Extend_FE) ÷ [999.0] SPACE (Other) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x3a, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x3a}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x3a, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x3a}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x3a, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x3a}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x3a, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x3a}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x3a, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x3a, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x3a, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x3a, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x3a, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x3a, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x3a, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x3a}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x3a, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x3a, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x2e}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2e, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2e}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2e, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2e}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x2e}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2e, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2e}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2e, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2e}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x2e}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2e, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2e, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x2e}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2e, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2e, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x2e}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2e, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2e, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2e, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x2e}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2e, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2e, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2c, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2c}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2c, 0x3a, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2c}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2c, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2c}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2c, 0x3a, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2c}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2c, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2c, 0x2e, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2c, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2c, 0x2e, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2c, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2c, 0x2c, 0x31}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x2c, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31}, []byte{0x2c}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x31, 0x2c, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x31, 0x5f, 0x61, 0x2c, 0x2c, 0x61}, - output: [][]byte{[]byte{0x31, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] DIGIT ONE (Numeric) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x3a, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x3a}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x3a, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x3a}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x3a, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x3a}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x3a, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x3a}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x3a, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x3a, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x3a, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x3a, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x3a, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x3a, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x3a, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x3a}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x3a, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x3a}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x3a, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x3a}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COLON (MidLetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2e, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x2e}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2e, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2e}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2e, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2e}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2e, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x2e}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2e, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2e}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2e, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2e}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2e, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x2e}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2e, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2e, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2e, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x2e}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2e, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2e, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2e, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x2e}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2e, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2e, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2e, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x2e}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2e, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2e}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2e, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2e}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2c, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2c}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2c, 0x3a, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2c}, []byte{0x3a}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2c, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2c}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2c, 0x3a, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2c}, []byte{0x3a}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COLON (MidLetter) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2c, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2c, 0x2e, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2e}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2c, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2c, 0x2e, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2e}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] FULL STOP (MidNumLet) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2c, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2c, 0x2c, 0x31}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2c}, []byte{0x31}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] DIGIT ONE (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61}, []byte{0x2c}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x31, 0x2c, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x31}, []byte{0x2c}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] DIGIT ONE (Numeric) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x5f, 0x61, 0x2c, 0x2c, 0x61}, - output: [][]byte{[]byte{0x61, 0x5f, 0x61}, []byte{0x2c}, []byte{0x2c}, []byte{0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (ALetter) × [13.1] LOW LINE (ExtendNumLet) × [13.2] LATIN SMALL LETTER A (ALetter) ÷ [999.0] COMMA (MidNum) ÷ [999.0] COMMA (MidNum) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [0.3]`, - }, -} -var unicodeSentenceTests = []struct { - input []byte - output [][]byte - comment string -}{ - { - input: []byte{0x1, 0x1}, - output: [][]byte{[]byte{0x1, 0x1}}, - comment: `÷ [0.2] (Other) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xd}, - output: [][]byte{[]byte{0x1, 0xd}}, - comment: `÷ [0.2] (Other) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xa}, - output: [][]byte{[]byte{0x1, 0xa}}, - comment: `÷ [0.2] (Other) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xc2, 0x85}, - output: [][]byte{[]byte{0x1, 0xc2, 0x85}}, - comment: `÷ [0.2] (Other) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x9}, - output: [][]byte{[]byte{0x1, 0x9}}, - comment: `÷ [0.2] (Other) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x61}, - output: [][]byte{[]byte{0x1, 0x61}}, - comment: `÷ [0.2] (Other) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x41}, - output: [][]byte{[]byte{0x1, 0x41}}, - comment: `÷ [0.2] (Other) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x1, 0xc6, 0xbb}}, - comment: `÷ [0.2] (Other) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x30}, - output: [][]byte{[]byte{0x1, 0x30}}, - comment: `÷ [0.2] (Other) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x2e}, - output: [][]byte{[]byte{0x1, 0x2e}}, - comment: `÷ [0.2] (Other) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x21}, - output: [][]byte{[]byte{0x1, 0x21}}, - comment: `÷ [0.2] (Other) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x22}, - output: [][]byte{[]byte{0x1, 0x22}}, - comment: `÷ [0.2] (Other) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0x2c}, - output: [][]byte{[]byte{0x1, 0x2c}}, - comment: `÷ [0.2] (Other) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xc2, 0xad}, - output: [][]byte{[]byte{0x1, 0xc2, 0xad}}, - comment: `÷ [0.2] (Other) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x80}, - output: [][]byte{[]byte{0x1, 0xcc, 0x80}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x1, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x1, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Other) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x1}, - output: [][]byte{[]byte{0xd}, []byte{0x1}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xd}, - output: [][]byte{[]byte{0xd}, []byte{0xd}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xa}, - output: [][]byte{[]byte{0xd, 0xa}}, - comment: `÷ [0.2] (CR) × [3.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xc2, 0x85}, - output: [][]byte{[]byte{0xd}, []byte{0xc2, 0x85}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x9}, - output: [][]byte{[]byte{0xd}, []byte{0x9}}, - comment: `÷ [0.2] (CR) ÷ [4.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x61}, - output: [][]byte{[]byte{0xd}, []byte{0x61}}, - comment: `÷ [0.2] (CR) ÷ [4.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x41}, - output: [][]byte{[]byte{0xd}, []byte{0x41}}, - comment: `÷ [0.2] (CR) ÷ [4.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xd}, []byte{0xc6, 0xbb}}, - comment: `÷ [0.2] (CR) ÷ [4.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x30}, - output: [][]byte{[]byte{0xd}, []byte{0x30}}, - comment: `÷ [0.2] (CR) ÷ [4.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x2e}, - output: [][]byte{[]byte{0xd}, []byte{0x2e}}, - comment: `÷ [0.2] (CR) ÷ [4.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x21}, - output: [][]byte{[]byte{0xd}, []byte{0x21}}, - comment: `÷ [0.2] (CR) ÷ [4.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x22}, - output: [][]byte{[]byte{0xd}, []byte{0x22}}, - comment: `÷ [0.2] (CR) ÷ [4.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0x2c}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xc2, 0xad}, - output: [][]byte{[]byte{0xd}, []byte{0xc2, 0xad}}, - comment: `÷ [0.2] (CR) ÷ [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xd, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xd}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (CR) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x1}, - output: [][]byte{[]byte{0xa}, []byte{0x1}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xd}, - output: [][]byte{[]byte{0xa}, []byte{0xd}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xa}, - output: [][]byte{[]byte{0xa}, []byte{0xa}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xc2, 0x85}, - output: [][]byte{[]byte{0xa}, []byte{0xc2, 0x85}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x9}, - output: [][]byte{[]byte{0xa}, []byte{0x9}}, - comment: `÷ [0.2] (LF) ÷ [4.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x61}, - output: [][]byte{[]byte{0xa}, []byte{0x61}}, - comment: `÷ [0.2] (LF) ÷ [4.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x41}, - output: [][]byte{[]byte{0xa}, []byte{0x41}}, - comment: `÷ [0.2] (LF) ÷ [4.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xa}, []byte{0xc6, 0xbb}}, - comment: `÷ [0.2] (LF) ÷ [4.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x30}, - output: [][]byte{[]byte{0xa}, []byte{0x30}}, - comment: `÷ [0.2] (LF) ÷ [4.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x2e}, - output: [][]byte{[]byte{0xa}, []byte{0x2e}}, - comment: `÷ [0.2] (LF) ÷ [4.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x21}, - output: [][]byte{[]byte{0xa}, []byte{0x21}}, - comment: `÷ [0.2] (LF) ÷ [4.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x22}, - output: [][]byte{[]byte{0xa}, []byte{0x22}}, - comment: `÷ [0.2] (LF) ÷ [4.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0x2c}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xc2, 0xad}, - output: [][]byte{[]byte{0xa}, []byte{0xc2, 0xad}}, - comment: `÷ [0.2] (LF) ÷ [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xa, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xa}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x1}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x1}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xd}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xd}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xa}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xa}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xc2, 0x85}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xc2, 0x85}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x9}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x9}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x61}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x61}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x41}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x41}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xc6, 0xbb}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x30}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x30}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x2e}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x2e}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x21}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x21}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x22}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x22}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0x2c}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0x2c}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xc2, 0xad}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x80}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0x85, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc2, 0x85}, []byte{0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Sep) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x1}, - output: [][]byte{[]byte{0x9, 0x1}}, - comment: `÷ [0.2] (Sp) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xd}, - output: [][]byte{[]byte{0x9, 0xd}}, - comment: `÷ [0.2] (Sp) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xa}, - output: [][]byte{[]byte{0x9, 0xa}}, - comment: `÷ [0.2] (Sp) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xc2, 0x85}, - output: [][]byte{[]byte{0x9, 0xc2, 0x85}}, - comment: `÷ [0.2] (Sp) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x9}, - output: [][]byte{[]byte{0x9, 0x9}}, - comment: `÷ [0.2] (Sp) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x61}, - output: [][]byte{[]byte{0x9, 0x61}}, - comment: `÷ [0.2] (Sp) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x41}, - output: [][]byte{[]byte{0x9, 0x41}}, - comment: `÷ [0.2] (Sp) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x9, 0xc6, 0xbb}}, - comment: `÷ [0.2] (Sp) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x30}, - output: [][]byte{[]byte{0x9, 0x30}}, - comment: `÷ [0.2] (Sp) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x2e}, - output: [][]byte{[]byte{0x9, 0x2e}}, - comment: `÷ [0.2] (Sp) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x21}, - output: [][]byte{[]byte{0x9, 0x21}}, - comment: `÷ [0.2] (Sp) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x22}, - output: [][]byte{[]byte{0x9, 0x22}}, - comment: `÷ [0.2] (Sp) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0x2c}, - output: [][]byte{[]byte{0x9, 0x2c}}, - comment: `÷ [0.2] (Sp) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xc2, 0xad}, - output: [][]byte{[]byte{0x9, 0xc2, 0xad}}, - comment: `÷ [0.2] (Sp) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x80}, - output: [][]byte{[]byte{0x9, 0xcc, 0x80}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x9, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x9, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x1}, - output: [][]byte{[]byte{0x61, 0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xd}, - output: [][]byte{[]byte{0x61, 0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xa}, - output: [][]byte{[]byte{0x61, 0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xc2, 0x85}, - output: [][]byte{[]byte{0x61, 0xc2, 0x85}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x9}, - output: [][]byte{[]byte{0x61, 0x9}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x61}, - output: [][]byte{[]byte{0x61, 0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x41}, - output: [][]byte{[]byte{0x61, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x61, 0xc6, 0xbb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x30}, - output: [][]byte{[]byte{0x61, 0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2e}, - output: [][]byte{[]byte{0x61, 0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x21}, - output: [][]byte{[]byte{0x61, 0x21}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x22}, - output: [][]byte{[]byte{0x61, 0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0x2c}, - output: [][]byte{[]byte{0x61, 0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x61, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x61, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN SMALL LETTER A (Lower) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x1}, - output: [][]byte{[]byte{0x41, 0x1}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xd}, - output: [][]byte{[]byte{0x41, 0xd}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xa}, - output: [][]byte{[]byte{0x41, 0xa}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xc2, 0x85}, - output: [][]byte{[]byte{0x41, 0xc2, 0x85}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x9}, - output: [][]byte{[]byte{0x41, 0x9}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x61}, - output: [][]byte{[]byte{0x41, 0x61}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x41}, - output: [][]byte{[]byte{0x41, 0x41}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x41, 0xc6, 0xbb}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x30}, - output: [][]byte{[]byte{0x41, 0x30}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x2e}, - output: [][]byte{[]byte{0x41, 0x2e}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x21}, - output: [][]byte{[]byte{0x41, 0x21}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x22}, - output: [][]byte{[]byte{0x41, 0x22}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0x2c}, - output: [][]byte{[]byte{0x41, 0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xc2, 0xad}, - output: [][]byte{[]byte{0x41, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x80}, - output: [][]byte{[]byte{0x41, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x41, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x41, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x1}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x1}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xd}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xd}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xa}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xa}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xc2, 0x85}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xc2, 0x85}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x9}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x9}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x61}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x61}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x41}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x41}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xc6, 0xbb}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x30}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x30}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x2e}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x2e}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x21}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x21}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x22}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x22}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0x2c}, - output: [][]byte{[]byte{0xc6, 0xbb, 0x2c}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc6, 0xbb, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc6, 0xbb, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] LATIN LETTER TWO WITH STROKE (OLetter) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x1}, - output: [][]byte{[]byte{0x30, 0x1}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xd}, - output: [][]byte{[]byte{0x30, 0xd}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xa}, - output: [][]byte{[]byte{0x30, 0xa}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xc2, 0x85}, - output: [][]byte{[]byte{0x30, 0xc2, 0x85}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x9}, - output: [][]byte{[]byte{0x30, 0x9}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x61}, - output: [][]byte{[]byte{0x30, 0x61}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x41}, - output: [][]byte{[]byte{0x30, 0x41}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x30, 0xc6, 0xbb}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x30}, - output: [][]byte{[]byte{0x30, 0x30}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x2e}, - output: [][]byte{[]byte{0x30, 0x2e}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x21}, - output: [][]byte{[]byte{0x30, 0x21}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x22}, - output: [][]byte{[]byte{0x30, 0x22}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0x2c}, - output: [][]byte{[]byte{0x30, 0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xc2, 0xad}, - output: [][]byte{[]byte{0x30, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x80}, - output: [][]byte{[]byte{0x30, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x30, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x30, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] DIGIT ZERO (Numeric) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x1}, - output: [][]byte{[]byte{0x2e}, []byte{0x1}}, - comment: `÷ [0.2] FULL STOP (ATerm) ÷ [11.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xd}, - output: [][]byte{[]byte{0x2e, 0xd}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [9.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xa}, - output: [][]byte{[]byte{0x2e, 0xa}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [9.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xc2, 0x85}, - output: [][]byte{[]byte{0x2e, 0xc2, 0x85}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [9.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x9}, - output: [][]byte{[]byte{0x2e, 0x9}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [9.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x61}, - output: [][]byte{[]byte{0x2e, 0x61}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [8.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x41}, - output: [][]byte{[]byte{0x2e}, []byte{0x41}}, - comment: `÷ [0.2] FULL STOP (ATerm) ÷ [11.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x2e}, []byte{0xc6, 0xbb}}, - comment: `÷ [0.2] FULL STOP (ATerm) ÷ [11.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88}, []byte{0xc6, 0xbb}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x30}, - output: [][]byte{[]byte{0x2e, 0x30}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [6.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [6.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x2e}, - output: [][]byte{[]byte{0x2e, 0x2e}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [8.1] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.1] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x21}, - output: [][]byte{[]byte{0x2e, 0x21}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [8.1] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.1] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x22}, - output: [][]byte{[]byte{0x2e, 0x22}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [9.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0x2c}, - output: [][]byte{[]byte{0x2e, 0x2c}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [8.1] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.1] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2e, 0xc2, 0xad}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x80}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2e, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2e, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] FULL STOP (ATerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x1}, - output: [][]byte{[]byte{0x21}, []byte{0x1}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) ÷ [11.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88}, []byte{0x1}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xd}, - output: [][]byte{[]byte{0x21, 0xd}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [9.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xa}, - output: [][]byte{[]byte{0x21, 0xa}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [9.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xc2, 0x85}, - output: [][]byte{[]byte{0x21, 0xc2, 0x85}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [9.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x9}, - output: [][]byte{[]byte{0x21, 0x9}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [9.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x61}, - output: [][]byte{[]byte{0x21}, []byte{0x61}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) ÷ [11.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88}, []byte{0x61}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x41}, - output: [][]byte{[]byte{0x21}, []byte{0x41}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) ÷ [11.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88}, []byte{0x41}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x21}, []byte{0xc6, 0xbb}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) ÷ [11.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88}, []byte{0xc6, 0xbb}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x30}, - output: [][]byte{[]byte{0x21}, []byte{0x30}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) ÷ [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88}, []byte{0x30}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x2e}, - output: [][]byte{[]byte{0x21, 0x2e}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [8.1] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.1] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x21}, - output: [][]byte{[]byte{0x21, 0x21}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [8.1] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.1] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x22}, - output: [][]byte{[]byte{0x21, 0x22}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [9.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [9.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x2c}, - output: [][]byte{[]byte{0x21, 0x2c}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [8.1] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.1] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xc2, 0xad}, - output: [][]byte{[]byte{0x21, 0xc2, 0xad}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x80}, - output: [][]byte{[]byte{0x21, 0xcc, 0x80}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x21, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x1}, - output: [][]byte{[]byte{0x22, 0x1}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xd}, - output: [][]byte{[]byte{0x22, 0xd}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xa}, - output: [][]byte{[]byte{0x22, 0xa}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xc2, 0x85}, - output: [][]byte{[]byte{0x22, 0xc2, 0x85}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x9}, - output: [][]byte{[]byte{0x22, 0x9}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x61}, - output: [][]byte{[]byte{0x22, 0x61}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x41}, - output: [][]byte{[]byte{0x22, 0x41}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x22, 0xc6, 0xbb}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x30}, - output: [][]byte{[]byte{0x22, 0x30}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x2e}, - output: [][]byte{[]byte{0x22, 0x2e}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x21}, - output: [][]byte{[]byte{0x22, 0x21}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x22}, - output: [][]byte{[]byte{0x22, 0x22}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0x2c}, - output: [][]byte{[]byte{0x22, 0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xc2, 0xad}, - output: [][]byte{[]byte{0x22, 0xc2, 0xad}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x80}, - output: [][]byte{[]byte{0x22, 0xcc, 0x80}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x22, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x22, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x1}, - output: [][]byte{[]byte{0x2c, 0x1}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xd}, - output: [][]byte{[]byte{0x2c, 0xd}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xa}, - output: [][]byte{[]byte{0x2c, 0xa}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xc2, 0x85}, - output: [][]byte{[]byte{0x2c, 0xc2, 0x85}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x9}, - output: [][]byte{[]byte{0x2c, 0x9}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x61}, - output: [][]byte{[]byte{0x2c, 0x61}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x41}, - output: [][]byte{[]byte{0x2c, 0x41}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x2c, 0xc6, 0xbb}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x30}, - output: [][]byte{[]byte{0x2c, 0x30}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x2e}, - output: [][]byte{[]byte{0x2c, 0x2e}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x21}, - output: [][]byte{[]byte{0x2c, 0x21}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x22}, - output: [][]byte{[]byte{0x2c, 0x22}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0x2c}, - output: [][]byte{[]byte{0x2c, 0x2c}}, - comment: `÷ [0.2] COMMA (SContinue) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2c, 0xc2, 0xad}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x80}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x2c, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0x2c, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] COMMA (SContinue) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x1}, - output: [][]byte{[]byte{0xc2, 0xad, 0x1}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xd}, - output: [][]byte{[]byte{0xc2, 0xad, 0xd}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xa}, - output: [][]byte{[]byte{0xc2, 0xad, 0xa}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xc2, 0x85}, - output: [][]byte{[]byte{0xc2, 0xad, 0xc2, 0x85}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x9}, - output: [][]byte{[]byte{0xc2, 0xad, 0x9}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x61}, - output: [][]byte{[]byte{0xc2, 0xad, 0x61}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x41}, - output: [][]byte{[]byte{0xc2, 0xad, 0x41}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xc2, 0xad, 0xc6, 0xbb}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x30}, - output: [][]byte{[]byte{0xc2, 0xad, 0x30}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x2e}, - output: [][]byte{[]byte{0xc2, 0xad, 0x2e}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x21}, - output: [][]byte{[]byte{0xc2, 0xad, 0x21}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x22}, - output: [][]byte{[]byte{0xc2, 0xad, 0x22}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad, 0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc2, 0xad, 0xc2, 0xad}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x80}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xc2, 0xad, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xc2, 0xad, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] SOFT HYPHEN (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x1}, - output: [][]byte{[]byte{0xcc, 0x80, 0x1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x1}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x1}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Other) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xd}, - output: [][]byte{[]byte{0xcc, 0x80, 0xd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xd}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xd}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (CR) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xa}, - output: [][]byte{[]byte{0xcc, 0x80, 0xa}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xa}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xa}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (LF) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xc2, 0x85}, - output: [][]byte{[]byte{0xcc, 0x80, 0xc2, 0x85}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xc2, 0x85}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xc2, 0x85}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sep) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x9}, - output: [][]byte{[]byte{0xcc, 0x80, 0x9}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x9}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x9}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x61}, - output: [][]byte{[]byte{0xcc, 0x80, 0x61}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x61}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x61}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN SMALL LETTER A (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x41}, - output: [][]byte{[]byte{0xcc, 0x80, 0x41}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x41}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x41}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER A (Upper) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xcc, 0x80, 0xc6, 0xbb}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xc6, 0xbb}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xc6, 0xbb}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN LETTER TWO WITH STROKE (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x30}, - output: [][]byte{[]byte{0xcc, 0x80, 0x30}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x30}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x30}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] DIGIT ZERO (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x2e}, - output: [][]byte{[]byte{0xcc, 0x80, 0x2e}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x2e}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x2e}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x21}, - output: [][]byte{[]byte{0xcc, 0x80, 0x21}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x21}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x21}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] EXCLAMATION MARK (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x22}, - output: [][]byte{[]byte{0xcc, 0x80, 0x22}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x22}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x22}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] QUOTATION MARK (Close) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80, 0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0x2c}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0x2c}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [12.0] COMMA (SContinue) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xc2, 0xad}, - output: [][]byte{[]byte{0xcc, 0x80, 0xc2, 0xad}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xc2, 0xad}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xc2, 0xad}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] SOFT HYPHEN (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0xcc, 0x80, 0xcc, 0x88, 0xcc, 0x80}, - output: [][]byte{[]byte{0xcc, 0x80, 0xcc, 0x88, 0xcc, 0x80}}, - comment: `÷ [0.2] COMBINING GRAVE ACCENT (Extend_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) ÷ [0.3]`, - }, - { - input: []byte{0x28, 0x22, 0x47, 0x6f, 0x2e, 0x22, 0x29, 0x20, 0x28, 0x48, 0x65, 0x20, 0x64, 0x69, 0x64, 0x2e, 0x29}, - output: [][]byte{[]byte{0x28, 0x22, 0x47, 0x6f, 0x2e, 0x22, 0x29, 0x20}, []byte{0x28, 0x48, 0x65, 0x20, 0x64, 0x69, 0x64, 0x2e, 0x29}}, - comment: `÷ [0.2] LEFT PARENTHESIS (Close) × [12.0] QUOTATION MARK (Close) × [12.0] LATIN CAPITAL LETTER G (Upper) × [12.0] LATIN SMALL LETTER O (Lower) × [12.0] FULL STOP (ATerm) × [9.0] QUOTATION MARK (Close) × [9.0] RIGHT PARENTHESIS (Close) × [9.0] SPACE (Sp) ÷ [11.0] LEFT PARENTHESIS (Close) × [12.0] LATIN CAPITAL LETTER H (Upper) × [12.0] LATIN SMALL LETTER E (Lower) × [12.0] SPACE (Sp) × [12.0] LATIN SMALL LETTER D (Lower) × [12.0] LATIN SMALL LETTER I (Lower) × [12.0] LATIN SMALL LETTER D (Lower) × [12.0] FULL STOP (ATerm) × [9.0] RIGHT PARENTHESIS (Close) ÷ [0.3]`, - }, - { - input: []byte{0x28, 0xe2, 0x80, 0x9c, 0x47, 0x6f, 0x3f, 0xe2, 0x80, 0x9d, 0x29, 0x20, 0x28, 0x48, 0x65, 0x20, 0x64, 0x69, 0x64, 0x2e, 0x29}, - output: [][]byte{[]byte{0x28, 0xe2, 0x80, 0x9c, 0x47, 0x6f, 0x3f, 0xe2, 0x80, 0x9d, 0x29, 0x20}, []byte{0x28, 0x48, 0x65, 0x20, 0x64, 0x69, 0x64, 0x2e, 0x29}}, - comment: `÷ [0.2] LEFT PARENTHESIS (Close) × [12.0] LEFT DOUBLE QUOTATION MARK (Close) × [12.0] LATIN CAPITAL LETTER G (Upper) × [12.0] LATIN SMALL LETTER O (Lower) × [12.0] QUESTION MARK (STerm) × [9.0] RIGHT DOUBLE QUOTATION MARK (Close) × [9.0] RIGHT PARENTHESIS (Close) × [9.0] SPACE (Sp) ÷ [11.0] LEFT PARENTHESIS (Close) × [12.0] LATIN CAPITAL LETTER H (Upper) × [12.0] LATIN SMALL LETTER E (Lower) × [12.0] SPACE (Sp) × [12.0] LATIN SMALL LETTER D (Lower) × [12.0] LATIN SMALL LETTER I (Lower) × [12.0] LATIN SMALL LETTER D (Lower) × [12.0] FULL STOP (ATerm) × [9.0] RIGHT PARENTHESIS (Close) ÷ [0.3]`, - }, - { - input: []byte{0x55, 0x2e, 0x53, 0x2e, 0x41, 0xcc, 0x80, 0x2e, 0x20, 0x69, 0x73}, - output: [][]byte{[]byte{0x55, 0x2e, 0x53, 0x2e, 0x41, 0xcc, 0x80, 0x2e, 0x20, 0x69, 0x73}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER U (Upper) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER S (Upper) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] FULL STOP (ATerm) × [8.0] SPACE (Sp) × [8.0] LATIN SMALL LETTER I (Lower) × [12.0] LATIN SMALL LETTER S (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x55, 0x2e, 0x53, 0x2e, 0x41, 0xcc, 0x80, 0x3f, 0x20, 0x48, 0x65}, - output: [][]byte{[]byte{0x55, 0x2e, 0x53, 0x2e, 0x41, 0xcc, 0x80, 0x3f, 0x20}, []byte{0x48, 0x65}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER U (Upper) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER S (Upper) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] QUESTION MARK (STerm) × [9.0] SPACE (Sp) ÷ [11.0] LATIN CAPITAL LETTER H (Upper) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x55, 0x2e, 0x53, 0x2e, 0x41, 0xcc, 0x80, 0x2e}, - output: [][]byte{[]byte{0x55, 0x2e, 0x53, 0x2e, 0x41, 0xcc, 0x80, 0x2e}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER U (Upper) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER S (Upper) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER A (Upper) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] FULL STOP (ATerm) ÷ [0.3]`, - }, - { - input: []byte{0x33, 0x2e, 0x34}, - output: [][]byte{[]byte{0x33, 0x2e, 0x34}}, - comment: `÷ [0.2] DIGIT THREE (Numeric) × [12.0] FULL STOP (ATerm) × [6.0] DIGIT FOUR (Numeric) ÷ [0.3]`, - }, - { - input: []byte{0x63, 0x2e, 0x64}, - output: [][]byte{[]byte{0x63, 0x2e, 0x64}}, - comment: `÷ [0.2] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [8.0] LATIN SMALL LETTER D (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x43, 0x2e, 0x64}, - output: [][]byte{[]byte{0x43, 0x2e, 0x64}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER C (Upper) × [12.0] FULL STOP (ATerm) × [8.0] LATIN SMALL LETTER D (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x63, 0x2e, 0x44}, - output: [][]byte{[]byte{0x63, 0x2e, 0x44}}, - comment: `÷ [0.2] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER D (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x43, 0x2e, 0x44}, - output: [][]byte{[]byte{0x43, 0x2e, 0x44}}, - comment: `÷ [0.2] LATIN CAPITAL LETTER C (Upper) × [12.0] FULL STOP (ATerm) × [7.0] LATIN CAPITAL LETTER D (Upper) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0x74, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0x74, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [8.0] RIGHT PARENTHESIS (Close) × [8.0] RIGHT SINGLE QUOTATION MARK (Close) × [8.0] NO-BREAK SPACE (Sp) × [8.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0x54, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0}, []byte{0x54, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [9.0] RIGHT PARENTHESIS (Close) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [9.0] NO-BREAK SPACE (Sp) ÷ [11.0] LATIN CAPITAL LETTER T (Upper) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0xe2, 0x80, 0x98, 0x28, 0x74, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0xe2, 0x80, 0x98, 0x28, 0x74, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [8.0] RIGHT PARENTHESIS (Close) × [8.0] RIGHT SINGLE QUOTATION MARK (Close) × [8.0] NO-BREAK SPACE (Sp) × [8.0] LEFT SINGLE QUOTATION MARK (Close) × [12.0] LEFT PARENTHESIS (Close) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0xe2, 0x80, 0x98, 0x28, 0x54, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0}, []byte{0xe2, 0x80, 0x98, 0x28, 0x54, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [9.0] RIGHT PARENTHESIS (Close) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [9.0] NO-BREAK SPACE (Sp) ÷ [11.0] LEFT SINGLE QUOTATION MARK (Close) × [12.0] LEFT PARENTHESIS (Close) × [12.0] LATIN CAPITAL LETTER T (Upper) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0xcc, 0x88, 0x74, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0xcc, 0x88, 0x74, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [8.0] RIGHT PARENTHESIS (Close) × [8.0] RIGHT SINGLE QUOTATION MARK (Close) × [8.0] NO-BREAK SPACE (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0xcc, 0x88, 0x54, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xc2, 0xa0, 0xcc, 0x88}, []byte{0x54, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [9.0] RIGHT PARENTHESIS (Close) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [9.0] NO-BREAK SPACE (Sp) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN CAPITAL LETTER T (Upper) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xcc, 0x88, 0x54, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xe2, 0x80, 0x99, 0xcc, 0x88}, []byte{0x54, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [9.0] RIGHT PARENTHESIS (Close) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN CAPITAL LETTER T (Upper) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xa, 0xcc, 0x88, 0x54, 0x68, 0x65}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0x29, 0xa}, []byte{0xcc, 0x88, 0x54, 0x68, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [9.0] RIGHT PARENTHESIS (Close) × [9.0] (LF) ÷ [4.0] COMBINING DIAERESIS (Extend_FE) × [12.0] LATIN CAPITAL LETTER T (Upper) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0x74, 0x68, 0x65, 0x20, 0x72, 0x65, 0x73, 0x70, 0x2e, 0x20, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x20, 0x61, 0x72, 0x65}, - output: [][]byte{[]byte{0x74, 0x68, 0x65, 0x20, 0x72, 0x65, 0x73, 0x70, 0x2e, 0x20, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x20, 0x61, 0x72, 0x65}}, - comment: `÷ [0.2] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER H (Lower) × [12.0] LATIN SMALL LETTER E (Lower) × [12.0] SPACE (Sp) × [12.0] LATIN SMALL LETTER R (Lower) × [12.0] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER S (Lower) × [12.0] LATIN SMALL LETTER P (Lower) × [12.0] FULL STOP (ATerm) × [8.0] SPACE (Sp) × [8.0] LATIN SMALL LETTER L (Lower) × [12.0] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER A (Lower) × [12.0] LATIN SMALL LETTER D (Lower) × [12.0] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER R (Lower) × [12.0] LATIN SMALL LETTER S (Lower) × [12.0] SPACE (Sp) × [12.0] LATIN SMALL LETTER A (Lower) × [12.0] LATIN SMALL LETTER R (Lower) × [12.0] LATIN SMALL LETTER E (Lower) ÷ [0.3]`, - }, - { - input: []byte{0xe5, 0xad, 0x97, 0x2e, 0xe5, 0xad, 0x97}, - output: [][]byte{[]byte{0xe5, 0xad, 0x97, 0x2e}, []byte{0xe5, 0xad, 0x97}}, - comment: `÷ [0.2] CJK UNIFIED IDEOGRAPH-5B57 (OLetter) × [12.0] FULL STOP (ATerm) ÷ [11.0] CJK UNIFIED IDEOGRAPH-5B57 (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0xe5, 0xae, 0x83}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e}, []byte{0xe5, 0xae, 0x83}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) ÷ [11.0] CJK UNIFIED IDEOGRAPH-5B83 (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x65, 0x74, 0x63, 0x2e, 0xe3, 0x80, 0x82}, - output: [][]byte{[]byte{0x65, 0x74, 0x63, 0x2e, 0xe3, 0x80, 0x82}}, - comment: `÷ [0.2] LATIN SMALL LETTER E (Lower) × [12.0] LATIN SMALL LETTER T (Lower) × [12.0] LATIN SMALL LETTER C (Lower) × [12.0] FULL STOP (ATerm) × [8.1] IDEOGRAPHIC FULL STOP (STerm) ÷ [0.3]`, - }, - { - input: []byte{0xe5, 0xad, 0x97, 0xe3, 0x80, 0x82, 0xe5, 0xae, 0x83}, - output: [][]byte{[]byte{0xe5, 0xad, 0x97, 0xe3, 0x80, 0x82}, []byte{0xe5, 0xae, 0x83}}, - comment: `÷ [0.2] CJK UNIFIED IDEOGRAPH-5B57 (OLetter) × [12.0] IDEOGRAPHIC FULL STOP (STerm) ÷ [11.0] CJK UNIFIED IDEOGRAPH-5B83 (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0x21, 0x20, 0x20}, - output: [][]byte{[]byte{0x21, 0x20, 0x20}}, - comment: `÷ [0.2] EXCLAMATION MARK (STerm) × [9.0] SPACE (Sp) × [10.0] SPACE (Sp) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x22, 0xe2, 0x81, 0xa0, 0x47, 0xe2, 0x81, 0xa0, 0x6f, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x22, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x48, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x69, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x22, 0xe2, 0x81, 0xa0, 0x47, 0xe2, 0x81, 0xa0, 0x6f, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x22, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0}, []byte{0x28, 0xe2, 0x81, 0xa0, 0x48, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x69, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LEFT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER G (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER O (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] LEFT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER H (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER D (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER I (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER D (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x9c, 0xe2, 0x81, 0xa0, 0x47, 0xe2, 0x81, 0xa0, 0x6f, 0xe2, 0x81, 0xa0, 0x3f, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x9d, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x48, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x69, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x9c, 0xe2, 0x81, 0xa0, 0x47, 0xe2, 0x81, 0xa0, 0x6f, 0xe2, 0x81, 0xa0, 0x3f, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x9d, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0}, []byte{0x28, 0xe2, 0x81, 0xa0, 0x48, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x69, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LEFT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LEFT DOUBLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER G (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER O (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] QUESTION MARK (STerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT DOUBLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] LEFT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER H (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER D (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER I (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER D (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x55, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x53, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x41, 0xe2, 0x81, 0xa0, 0xcc, 0x80, 0x2e, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x69, 0xe2, 0x81, 0xa0, 0x73, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x55, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x53, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x41, 0xe2, 0x81, 0xa0, 0xcc, 0x80, 0x2e, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x69, 0xe2, 0x81, 0xa0, 0x73, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER U (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER S (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER A (Upper) × [5.0] WORD JOINER (Format_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [8.0] LATIN SMALL LETTER I (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER S (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x55, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x53, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x41, 0xe2, 0x81, 0xa0, 0xcc, 0x80, 0x3f, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x48, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x55, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x53, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x41, 0xe2, 0x81, 0xa0, 0xcc, 0x80, 0x3f, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0}, []byte{0x48, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER U (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER S (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER A (Upper) × [5.0] WORD JOINER (Format_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] QUESTION MARK (STerm) × [5.0] WORD JOINER (Format_FE) × [9.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] LATIN CAPITAL LETTER H (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x55, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x53, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x41, 0xe2, 0x81, 0xa0, 0xcc, 0x80, 0x2e, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x55, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x53, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x41, 0xe2, 0x81, 0xa0, 0xcc, 0x80, 0x2e, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER U (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER S (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER A (Upper) × [5.0] WORD JOINER (Format_FE) × [5.0] COMBINING GRAVE ACCENT (Extend_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x33, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x34, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x33, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x34, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] DIGIT THREE (Numeric) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [6.0] DIGIT FOUR (Numeric) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.0] LATIN SMALL LETTER D (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER C (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.0] LATIN SMALL LETTER D (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER D (Upper) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x43, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x44, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER C (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [7.0] LATIN CAPITAL LETTER D (Upper) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [8.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [8.0] NO-BREAK SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [8.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0}, []byte{0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] NO-BREAK SPACE (Sp) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] LATIN CAPITAL LETTER T (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x98, 0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x98, 0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [8.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [8.0] NO-BREAK SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [8.0] LEFT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LEFT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x98, 0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0}, []byte{0xe2, 0x80, 0x98, 0xe2, 0x81, 0xa0, 0x28, 0xe2, 0x81, 0xa0, 0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] NO-BREAK SPACE (Sp) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] LEFT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LEFT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER T (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [8.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [8.0] NO-BREAK SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [8.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xc2, 0xa0, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] NO-BREAK SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN CAPITAL LETTER T (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xe2, 0x80, 0x99, 0xe2, 0x81, 0xa0, 0xcc, 0x88}, []byte{0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT SINGLE QUOTATION MARK (Close) × [5.0] WORD JOINER (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) ÷ [11.0] LATIN CAPITAL LETTER T (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xa, 0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xe2, 0x81, 0xa0, 0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x29, 0xe2, 0x81, 0xa0, 0xa}, []byte{0xe2, 0x81, 0xa0, 0xcc, 0x88, 0xe2, 0x81, 0xa0, 0x54, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [9.0] RIGHT PARENTHESIS (Close) × [5.0] WORD JOINER (Format_FE) × [9.0] (LF) ÷ [4.0] WORD JOINER (Format_FE) × [5.0] COMBINING DIAERESIS (Extend_FE) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN CAPITAL LETTER T (Upper) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x72, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x73, 0xe2, 0x81, 0xa0, 0x70, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x6c, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x72, 0xe2, 0x81, 0xa0, 0x73, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x72, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x68, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x72, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x73, 0xe2, 0x81, 0xa0, 0x70, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x6c, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x64, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x72, 0xe2, 0x81, 0xa0, 0x73, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x61, 0xe2, 0x81, 0xa0, 0x72, 0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER H (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER R (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER S (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER P (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [8.0] LATIN SMALL LETTER L (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER A (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER D (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER R (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER S (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER A (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER R (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0xe5, 0xad, 0x97, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0xe5, 0xad, 0x97, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0xe5, 0xad, 0x97, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0}, []byte{0xe5, 0xad, 0x97, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] CJK UNIFIED IDEOGRAPH-5B57 (OLetter) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] CJK UNIFIED IDEOGRAPH-5B57 (OLetter) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0xe5, 0xae, 0x83, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0}, []byte{0xe5, 0xae, 0x83, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] CJK UNIFIED IDEOGRAPH-5B83 (OLetter) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0xe3, 0x80, 0x82, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x65, 0xe2, 0x81, 0xa0, 0x74, 0xe2, 0x81, 0xa0, 0x63, 0xe2, 0x81, 0xa0, 0x2e, 0xe2, 0x81, 0xa0, 0xe3, 0x80, 0x82, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER E (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER T (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] LATIN SMALL LETTER C (Lower) × [5.0] WORD JOINER (Format_FE) × [12.0] FULL STOP (ATerm) × [5.0] WORD JOINER (Format_FE) × [8.1] IDEOGRAPHIC FULL STOP (STerm) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0xe5, 0xad, 0x97, 0xe2, 0x81, 0xa0, 0xe3, 0x80, 0x82, 0xe2, 0x81, 0xa0, 0xe5, 0xae, 0x83, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0xe5, 0xad, 0x97, 0xe2, 0x81, 0xa0, 0xe3, 0x80, 0x82, 0xe2, 0x81, 0xa0}, []byte{0xe5, 0xae, 0x83, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] CJK UNIFIED IDEOGRAPH-5B57 (OLetter) × [5.0] WORD JOINER (Format_FE) × [12.0] IDEOGRAPHIC FULL STOP (STerm) × [5.0] WORD JOINER (Format_FE) ÷ [11.0] CJK UNIFIED IDEOGRAPH-5B83 (OLetter) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xe2, 0x81, 0xa0, 0x21, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}, - output: [][]byte{[]byte{0xe2, 0x81, 0xa0, 0x21, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0x20, 0xe2, 0x81, 0xa0, 0xe2, 0x81, 0xa0}}, - comment: `÷ [0.2] WORD JOINER (Format_FE) × [12.0] EXCLAMATION MARK (STerm) × [5.0] WORD JOINER (Format_FE) × [9.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [10.0] SPACE (Sp) × [5.0] WORD JOINER (Format_FE) × [5.0] WORD JOINER (Format_FE) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Other) × [12.0] REGIONAL INDICATOR SYMBOL LETTER B (Other) × [12.0] REGIONAL INDICATOR SYMBOL LETTER C (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa7, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Other) × [5.0] ZERO WIDTH JOINER (Extend_FE) × [12.0] REGIONAL INDICATOR SYMBOL LETTER B (Other) × [12.0] REGIONAL INDICATOR SYMBOL LETTER C (Other) ÷ [0.3]`, - }, - { - input: []byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa8}, - output: [][]byte{[]byte{0xf0, 0x9f, 0x87, 0xa6, 0xf0, 0x9f, 0x87, 0xa7, 0xe2, 0x80, 0x8d, 0xf0, 0x9f, 0x87, 0xa8}}, - comment: `÷ [0.2] REGIONAL INDICATOR SYMBOL LETTER A (Other) × [12.0] REGIONAL INDICATOR SYMBOL LETTER B (Other) × [5.0] ZERO WIDTH JOINER (Extend_FE) × [12.0] REGIONAL INDICATOR SYMBOL LETTER C (Other) ÷ [0.3]`, - }, - { - input: []byte{0x20, 0xe2, 0x80, 0x8d, 0xd9, 0x86}, - output: [][]byte{[]byte{0x20, 0xe2, 0x80, 0x8d, 0xd9, 0x86}}, - comment: `÷ [0.2] SPACE (Sp) × [5.0] ZERO WIDTH JOINER (Extend_FE) × [12.0] ARABIC LETTER NOON (OLetter) ÷ [0.3]`, - }, - { - input: []byte{0xd9, 0x86, 0xe2, 0x80, 0x8d, 0x20}, - output: [][]byte{[]byte{0xd9, 0x86, 0xe2, 0x80, 0x8d, 0x20}}, - comment: `÷ [0.2] ARABIC LETTER NOON (OLetter) × [5.0] ZERO WIDTH JOINER (Extend_FE) × [12.0] SPACE (Sp) ÷ [0.3]`, - }, -} diff --git a/vendor/github.com/boltdb/bolt/README.md b/vendor/github.com/boltdb/bolt/README.md index 8a21c6ee..d7f80e97 100644 --- a/vendor/github.com/boltdb/bolt/README.md +++ b/vendor/github.com/boltdb/bolt/README.md @@ -1,4 +1,4 @@ -Bolt [![Build Status](https://drone.io/github.com/boltdb/bolt/status.png)](https://drone.io/github.com/boltdb/bolt/latest) [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg) +Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg) ==== Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas] @@ -15,11 +15,11 @@ and setting values. That's it. ## Project Status -Bolt is stable and the API is fixed. Full unit test coverage and randomized -black box testing are used to ensure database consistency and thread safety. -Bolt is currently in high-load production environments serving databases as -large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed -services every day. +Bolt is stable, the API is fixed, and the file format is fixed. Full unit +test coverage and randomized black box testing are used to ensure database +consistency and thread safety. Bolt is currently used in high-load production +environments serving databases as large as 1TB. Many companies such as +Shopify and Heroku use Bolt-backed services every day. ## Table of Contents @@ -209,7 +209,7 @@ and then safely close your transaction if an error is returned. This is the recommended way to use Bolt transactions. However, sometimes you may want to manually start and end your transactions. -You can use the `Tx.Begin()` function directly but **please** be sure to close +You can use the `DB.Begin()` function directly but **please** be sure to close the transaction. ```go @@ -313,7 +313,7 @@ func (s *Store) CreateUser(u *User) error { // Generate ID for the user. // This returns an error only if the Tx is closed or not writeable. // That can't happen in an Update() call so I ignore the error check. - id, _ = b.NextSequence() + id, _ := b.NextSequence() u.ID = int(id) // Marshal user data into bytes. @@ -395,7 +395,7 @@ db.View(func(tx *bolt.Tx) error { c := tx.Bucket([]byte("MyBucket")).Cursor() prefix := []byte("1234") - for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() { + for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { fmt.Printf("key=%s, value=%s\n", k, v) } @@ -427,6 +427,8 @@ db.View(func(tx *bolt.Tx) error { }) ``` +Note that, while RFC3339 is sortable, the Golang implementation of RFC3339Nano does not use a fixed number of digits after the decimal point and is therefore not sortable. + #### ForEach() @@ -437,7 +439,7 @@ all the keys in a bucket: db.View(func(tx *bolt.Tx) error { // Assume bucket exists and has keys b := tx.Bucket([]byte("MyBucket")) - + b.ForEach(func(k, v []byte) error { fmt.Printf("key=%s, value=%s\n", k, v) return nil @@ -446,6 +448,10 @@ db.View(func(tx *bolt.Tx) error { }) ``` +Please note that keys and values in `ForEach()` are only valid while +the transaction is open. If you need to use a key or value outside of +the transaction, you must use `copy()` to copy it to another byte +slice. ### Nested buckets @@ -458,6 +464,55 @@ func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error) func (*Bucket) DeleteBucket(key []byte) error ``` +Say you had a multi-tenant application where the root level bucket was the account bucket. Inside of this bucket was a sequence of accounts which themselves are buckets. And inside the sequence bucket you could have many buckets pertaining to the Account itself (Users, Notes, etc) isolating the information into logical groupings. + +```go + +// createUser creates a new user in the given account. +func createUser(accountID int, u *User) error { + // Start the transaction. + tx, err := db.Begin(true) + if err != nil { + return err + } + defer tx.Rollback() + + // Retrieve the root bucket for the account. + // Assume this has already been created when the account was set up. + root := tx.Bucket([]byte(strconv.FormatUint(accountID, 10))) + + // Setup the users bucket. + bkt, err := root.CreateBucketIfNotExists([]byte("USERS")) + if err != nil { + return err + } + + // Generate an ID for the new user. + userID, err := bkt.NextSequence() + if err != nil { + return err + } + u.ID = userID + + // Marshal and save the encoded user. + if buf, err := json.Marshal(u); err != nil { + return err + } else if err := bkt.Put([]byte(strconv.FormatUint(u.ID, 10)), buf); err != nil { + return err + } + + // Commit the transaction. + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +``` + + + ### Database backups @@ -555,7 +610,7 @@ if err != nil { Bolt is able to run on mobile devices by leveraging the binding feature of the [gomobile](https://github.com/golang/mobile) tool. Create a struct that will contain your database logic and a reference to a `*bolt.DB` with a initializing -contstructor that takes in a filepath where the database file will be stored. +constructor that takes in a filepath where the database file will be stored. Neither Android nor iOS require extra permissions or cleanup from using this method. ```go @@ -617,7 +672,7 @@ Boltmobiledemo.BoltDB boltDB = Boltmobiledemo.NewBoltDB(path) { NSURL* URL= [NSURL fileURLWithPath: filePathString]; assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]); - + NSError *error = nil; BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES] forKey: NSURLIsExcludedFromBackupKey error: &error]; @@ -713,6 +768,9 @@ Here are a few things to note when evaluating and using Bolt: can be reused by a new page or can be unmapped from virtual memory and you'll see an `unexpected fault address` panic when accessing it. +* Bolt uses an exclusive write lock on the database file so it cannot be + shared by multiple processes. + * Be careful when using `Bucket.FillPercent`. Setting a high fill percent for buckets that have random inserts will cause your database to have very poor page utilization. @@ -805,6 +863,7 @@ them via pull request. Below is a list of public, open source projects that use Bolt: +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. * [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. * [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. @@ -823,7 +882,6 @@ Below is a list of public, open source projects that use Bolt: * [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. * [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [SkyDB](https://github.com/skydb/sky) - Behavioral analytics database. * [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. * [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. * [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. @@ -839,5 +897,19 @@ Below is a list of public, open source projects that use Bolt: * [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. * [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. +* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. +* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. +* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. +* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. +* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. +* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains +* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. +* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. +* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. +* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies +* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/boltdb/bolt/bolt_386.go b/vendor/github.com/boltdb/bolt/bolt_386.go index e659bfb9..820d533c 100644 --- a/vendor/github.com/boltdb/bolt/bolt_386.go +++ b/vendor/github.com/boltdb/bolt/bolt_386.go @@ -5,3 +5,6 @@ const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_amd64.go b/vendor/github.com/boltdb/bolt/bolt_amd64.go index cca6b7eb..98fafdb4 100644 --- a/vendor/github.com/boltdb/bolt/bolt_amd64.go +++ b/vendor/github.com/boltdb/bolt/bolt_amd64.go @@ -5,3 +5,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_arm.go b/vendor/github.com/boltdb/bolt/bolt_arm.go index e659bfb9..7e5cb4b9 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm.go +++ b/vendor/github.com/boltdb/bolt/bolt_arm.go @@ -1,7 +1,28 @@ package bolt +import "unsafe" + // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned bool + +func init() { + // Simple check to see whether this arch handles unaligned load/stores + // correctly. + + // ARM9 and older devices require load/stores to be from/to aligned + // addresses. If not, the lower 2 bits are cleared and that address is + // read in a jumbled up order. + + // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html + + raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11} + val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2)) + + brokenUnaligned = val != 0x11222211 +} diff --git a/vendor/github.com/boltdb/bolt/bolt_arm64.go b/vendor/github.com/boltdb/bolt/bolt_arm64.go index 6d230935..b26d84f9 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm64.go +++ b/vendor/github.com/boltdb/bolt/bolt_arm64.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc.go b/vendor/github.com/boltdb/bolt/bolt_ppc.go new file mode 100644 index 00000000..645ddc3e --- /dev/null +++ b/vendor/github.com/boltdb/bolt/bolt_ppc.go @@ -0,0 +1,9 @@ +// +build ppc + +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0x7FFFFFFF // 2GB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0xFFFFFFF diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64.go b/vendor/github.com/boltdb/bolt/bolt_ppc64.go new file mode 100644 index 00000000..9331d977 --- /dev/null +++ b/vendor/github.com/boltdb/bolt/bolt_ppc64.go @@ -0,0 +1,12 @@ +// +build ppc64 + +package bolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go b/vendor/github.com/boltdb/bolt/bolt_ppc64le.go index 8351e129..8c143bc5 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go +++ b/vendor/github.com/boltdb/bolt/bolt_ppc64le.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_s390x.go b/vendor/github.com/boltdb/bolt/bolt_s390x.go index f4dd26bb..d7c39af9 100644 --- a/vendor/github.com/boltdb/bolt/bolt_s390x.go +++ b/vendor/github.com/boltdb/bolt/bolt_s390x.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_unix.go b/vendor/github.com/boltdb/bolt/bolt_unix.go index 4b0723aa..cad62dda 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix.go +++ b/vendor/github.com/boltdb/bolt/bolt_unix.go @@ -11,7 +11,7 @@ import ( ) // flock acquires an advisory lock on a file descriptor. -func flock(f *os.File, exclusive bool, timeout time.Duration) error { +func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { var t time.Time for { // If we're beyond our timeout then return an error. @@ -27,7 +27,7 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error { } // Otherwise attempt to obtain an exclusive lock. - err := syscall.Flock(int(f.Fd()), flag|syscall.LOCK_NB) + err := syscall.Flock(int(db.file.Fd()), flag|syscall.LOCK_NB) if err == nil { return nil } else if err != syscall.EWOULDBLOCK { @@ -40,8 +40,8 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error { } // funlock releases an advisory lock on a file descriptor. -func funlock(f *os.File) error { - return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) +func funlock(db *DB) error { + return syscall.Flock(int(db.file.Fd()), syscall.LOCK_UN) } // mmap memory maps a DB's data file. diff --git a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go b/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go index 1c4e48d6..307bf2b3 100644 --- a/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go +++ b/vendor/github.com/boltdb/bolt/bolt_unix_solaris.go @@ -11,7 +11,7 @@ import ( ) // flock acquires an advisory lock on a file descriptor. -func flock(f *os.File, exclusive bool, timeout time.Duration) error { +func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { var t time.Time for { // If we're beyond our timeout then return an error. @@ -32,7 +32,7 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error { } else { lock.Type = syscall.F_RDLCK } - err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &lock) + err := syscall.FcntlFlock(db.file.Fd(), syscall.F_SETLK, &lock) if err == nil { return nil } else if err != syscall.EAGAIN { @@ -45,13 +45,13 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error { } // funlock releases an advisory lock on a file descriptor. -func funlock(f *os.File) error { +func funlock(db *DB) error { var lock syscall.Flock_t lock.Start = 0 lock.Len = 0 lock.Type = syscall.F_UNLCK lock.Whence = 0 - return syscall.FcntlFlock(uintptr(f.Fd()), syscall.F_SETLK, &lock) + return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock) } // mmap memory maps a DB's data file. diff --git a/vendor/github.com/boltdb/bolt/bolt_windows.go b/vendor/github.com/boltdb/bolt/bolt_windows.go index 91c4968f..b00fb072 100644 --- a/vendor/github.com/boltdb/bolt/bolt_windows.go +++ b/vendor/github.com/boltdb/bolt/bolt_windows.go @@ -16,6 +16,8 @@ var ( ) const ( + lockExt = ".lock" + // see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx flagLockExclusive = 2 flagLockFailImmediately = 1 @@ -46,7 +48,16 @@ func fdatasync(db *DB) error { } // flock acquires an advisory lock on a file descriptor. -func flock(f *os.File, exclusive bool, timeout time.Duration) error { +func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { + // Create a separate lock file on windows because a process + // cannot share an exclusive lock on the same file. This is + // needed during Tx.WriteTo(). + f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode) + if err != nil { + return err + } + db.lockfile = f + var t time.Time for { // If we're beyond our timeout then return an error. @@ -62,7 +73,7 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error { flag |= flagLockExclusive } - err := lockFileEx(syscall.Handle(f.Fd()), flag, 0, 1, 0, &syscall.Overlapped{}) + err := lockFileEx(syscall.Handle(db.lockfile.Fd()), flag, 0, 1, 0, &syscall.Overlapped{}) if err == nil { return nil } else if err != errLockViolation { @@ -75,8 +86,11 @@ func flock(f *os.File, exclusive bool, timeout time.Duration) error { } // funlock releases an advisory lock on a file descriptor. -func funlock(f *os.File) error { - return unlockFileEx(syscall.Handle(f.Fd()), 0, 1, 0, &syscall.Overlapped{}) +func funlock(db *DB) error { + err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{}) + db.lockfile.Close() + os.Remove(db.path + lockExt) + return err } // mmap memory maps a DB's data file. diff --git a/vendor/github.com/boltdb/bolt/bucket.go b/vendor/github.com/boltdb/bolt/bucket.go index d2f8c524..0c5bf274 100644 --- a/vendor/github.com/boltdb/bolt/bucket.go +++ b/vendor/github.com/boltdb/bolt/bucket.go @@ -130,9 +130,17 @@ func (b *Bucket) Bucket(name []byte) *Bucket { func (b *Bucket) openBucket(value []byte) *Bucket { var child = newBucket(b.tx) + // If unaligned load/stores are broken on this arch and value is + // unaligned simply clone to an aligned byte array. + unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0 + + if unaligned { + value = cloneBytes(value) + } + // If this is a writable transaction then we need to copy the bucket entry. // Read-only transactions can point directly at the mmap entry. - if b.tx.writable { + if b.tx.writable && !unaligned { child.bucket = &bucket{} *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) } else { @@ -167,9 +175,8 @@ func (b *Bucket) CreateBucket(key []byte) (*Bucket, error) { if bytes.Equal(key, k) { if (flags & bucketLeafFlag) != 0 { return nil, ErrBucketExists - } else { - return nil, ErrIncompatibleValue } + return nil, ErrIncompatibleValue } // Create empty, inline bucket. @@ -329,6 +336,28 @@ func (b *Bucket) Delete(key []byte) error { return nil } +// Sequence returns the current integer for the bucket without incrementing it. +func (b *Bucket) Sequence() uint64 { return b.bucket.sequence } + +// SetSequence updates the sequence number for the bucket. +func (b *Bucket) SetSequence(v uint64) error { + if b.tx.db == nil { + return ErrTxClosed + } else if !b.Writable() { + return ErrTxNotWritable + } + + // Materialize the root node if it hasn't been already so that the + // bucket will be saved during commit. + if b.rootNode == nil { + _ = b.node(b.root, nil) + } + + // Increment and return the sequence. + b.bucket.sequence = v + return nil +} + // NextSequence returns an autoincrementing integer for the bucket. func (b *Bucket) NextSequence() (uint64, error) { if b.tx.db == nil { diff --git a/vendor/github.com/boltdb/bolt/bucket_test.go b/vendor/github.com/boltdb/bolt/bucket_test.go deleted file mode 100644 index a02c3671..00000000 --- a/vendor/github.com/boltdb/bolt/bucket_test.go +++ /dev/null @@ -1,1827 +0,0 @@ -package bolt_test - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "log" - "math/rand" - "os" - "strconv" - "strings" - "testing" - "testing/quick" - - "github.com/boltdb/bolt" -) - -// Ensure that a bucket that gets a non-existent key returns nil. -func TestBucket_Get_NonExistent(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if v := b.Get([]byte("foo")); v != nil { - t.Fatal("expected nil value") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can read a value that is not flushed yet. -func TestBucket_Get_FromNode(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if v := b.Get([]byte("foo")); !bytes.Equal(v, []byte("bar")) { - t.Fatalf("unexpected value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket retrieved via Get() returns a nil. -func TestBucket_Get_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if _, err := tx.Bucket([]byte("widgets")).CreateBucket([]byte("foo")); err != nil { - t.Fatal(err) - } - - if tx.Bucket([]byte("widgets")).Get([]byte("foo")) != nil { - t.Fatal("expected nil value") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can write a key/value. -func TestBucket_Put(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - - v := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - if !bytes.Equal([]byte("bar"), v) { - t.Fatalf("unexpected value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can rewrite a key in the same transaction. -func TestBucket_Put_Repeat(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("baz")); err != nil { - t.Fatal(err) - } - - value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - if !bytes.Equal([]byte("baz"), value) { - t.Fatalf("unexpected value: %v", value) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can write a bunch of large values. -func TestBucket_Put_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - count, factor := 100, 200 - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - for i := 1; i < count; i++ { - if err := b.Put([]byte(strings.Repeat("0", i*factor)), []byte(strings.Repeat("X", (count-i)*factor))); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for i := 1; i < count; i++ { - value := b.Get([]byte(strings.Repeat("0", i*factor))) - if !bytes.Equal(value, []byte(strings.Repeat("X", (count-i)*factor))) { - t.Fatalf("unexpected value: %v", value) - } - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a database can perform multiple large appends safely. -func TestDB_Put_VeryLarge(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - n, batchN := 400000, 200000 - ksize, vsize := 8, 500 - - db := MustOpenDB() - defer db.MustClose() - - for i := 0; i < n; i += batchN { - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - for j := 0; j < batchN; j++ { - k, v := make([]byte, ksize), make([]byte, vsize) - binary.BigEndian.PutUint32(k, uint32(i+j)) - if err := b.Put(k, v); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - } -} - -// Ensure that a setting a value on a key with a bucket value returns an error. -func TestBucket_Put_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b0, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if _, err := tx.Bucket([]byte("widgets")).CreateBucket([]byte("foo")); err != nil { - t.Fatal(err) - } - if err := b0.Put([]byte("foo"), []byte("bar")); err != bolt.ErrIncompatibleValue { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a setting a value while the transaction is closed returns an error. -func TestBucket_Put_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - - if err := b.Put([]byte("foo"), []byte("bar")); err != bolt.ErrTxClosed { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that setting a value on a read-only bucket returns an error. -func TestBucket_Put_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - if err := b.Put([]byte("foo"), []byte("bar")); err != bolt.ErrTxNotWritable { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can delete an existing key. -func TestBucket_Delete(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if err := b.Delete([]byte("foo")); err != nil { - t.Fatal(err) - } - if v := b.Get([]byte("foo")); v != nil { - t.Fatalf("unexpected value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that deleting a large set of keys will work correctly. -func TestBucket_Delete_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 100; i++ { - if err := b.Put([]byte(strconv.Itoa(i)), []byte(strings.Repeat("*", 1024))); err != nil { - t.Fatal(err) - } - } - - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for i := 0; i < 100; i++ { - if err := b.Delete([]byte(strconv.Itoa(i))); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for i := 0; i < 100; i++ { - if v := b.Get([]byte(strconv.Itoa(i))); v != nil { - t.Fatalf("unexpected value: %v, i=%d", v, i) - } - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Deleting a very large list of keys will cause the freelist to use overflow. -func TestBucket_Delete_FreelistOverflow(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - db := MustOpenDB() - defer db.MustClose() - - k := make([]byte, 16) - for i := uint64(0); i < 10000; i++ { - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte("0")) - if err != nil { - t.Fatalf("bucket error: %s", err) - } - - for j := uint64(0); j < 1000; j++ { - binary.BigEndian.PutUint64(k[:8], i) - binary.BigEndian.PutUint64(k[8:], j) - if err := b.Put(k, nil); err != nil { - t.Fatalf("put error: %s", err) - } - } - - return nil - }); err != nil { - t.Fatal(err) - } - } - - // Delete all of them in one large transaction - if err := db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("0")) - c := b.Cursor() - for k, _ := c.First(); k != nil; k, _ = c.Next() { - if err := c.Delete(); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that accessing and updating nested buckets is ok across transactions. -func TestBucket_Nested(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - // Create a widgets bucket. - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - // Create a widgets/foo bucket. - _, err = b.CreateBucket([]byte("foo")) - if err != nil { - t.Fatal(err) - } - - // Create a widgets/bar key. - if err := b.Put([]byte("bar"), []byte("0000")); err != nil { - t.Fatal(err) - } - - return nil - }); err != nil { - t.Fatal(err) - } - db.MustCheck() - - // Update widgets/bar. - if err := db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - if err := b.Put([]byte("bar"), []byte("xxxx")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - db.MustCheck() - - // Cause a split. - if err := db.Update(func(tx *bolt.Tx) error { - var b = tx.Bucket([]byte("widgets")) - for i := 0; i < 10000; i++ { - if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - db.MustCheck() - - // Insert into widgets/foo/baz. - if err := db.Update(func(tx *bolt.Tx) error { - var b = tx.Bucket([]byte("widgets")) - if err := b.Bucket([]byte("foo")).Put([]byte("baz"), []byte("yyyy")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - db.MustCheck() - - // Verify. - if err := db.View(func(tx *bolt.Tx) error { - var b = tx.Bucket([]byte("widgets")) - if v := b.Bucket([]byte("foo")).Get([]byte("baz")); !bytes.Equal(v, []byte("yyyy")) { - t.Fatalf("unexpected value: %v", v) - } - if v := b.Get([]byte("bar")); !bytes.Equal(v, []byte("xxxx")) { - t.Fatalf("unexpected value: %v", v) - } - for i := 0; i < 10000; i++ { - if v := b.Get([]byte(strconv.Itoa(i))); !bytes.Equal(v, []byte(strconv.Itoa(i))) { - t.Fatalf("unexpected value: %v", v) - } - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that deleting a bucket using Delete() returns an error. -func TestBucket_Delete_Bucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if _, err := b.CreateBucket([]byte("foo")); err != nil { - t.Fatal(err) - } - if err := b.Delete([]byte("foo")); err != bolt.ErrIncompatibleValue { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that deleting a key on a read-only bucket returns an error. -func TestBucket_Delete_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - if err := tx.Bucket([]byte("widgets")).Delete([]byte("foo")); err != bolt.ErrTxNotWritable { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a deleting value while the transaction is closed returns an error. -func TestBucket_Delete_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - if err := b.Delete([]byte("foo")); err != bolt.ErrTxClosed { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that deleting a bucket causes nested buckets to be deleted. -func TestBucket_DeleteBucket_Nested(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - widgets, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - foo, err := widgets.CreateBucket([]byte("foo")) - if err != nil { - t.Fatal(err) - } - - bar, err := foo.CreateBucket([]byte("bar")) - if err != nil { - t.Fatal(err) - } - if err := bar.Put([]byte("baz"), []byte("bat")); err != nil { - t.Fatal(err) - } - if err := tx.Bucket([]byte("widgets")).DeleteBucket([]byte("foo")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that deleting a bucket causes nested buckets to be deleted after they have been committed. -func TestBucket_DeleteBucket_Nested2(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - widgets, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - foo, err := widgets.CreateBucket([]byte("foo")) - if err != nil { - t.Fatal(err) - } - - bar, err := foo.CreateBucket([]byte("bar")) - if err != nil { - t.Fatal(err) - } - - if err := bar.Put([]byte("baz"), []byte("bat")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - widgets := tx.Bucket([]byte("widgets")) - if widgets == nil { - t.Fatal("expected widgets bucket") - } - - foo := widgets.Bucket([]byte("foo")) - if foo == nil { - t.Fatal("expected foo bucket") - } - - bar := foo.Bucket([]byte("bar")) - if bar == nil { - t.Fatal("expected bar bucket") - } - - if v := bar.Get([]byte("baz")); !bytes.Equal(v, []byte("bat")) { - t.Fatalf("unexpected value: %v", v) - } - if err := tx.DeleteBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - if tx.Bucket([]byte("widgets")) != nil { - t.Fatal("expected bucket to be deleted") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that deleting a child bucket with multiple pages causes all pages to get collected. -// NOTE: Consistency check in bolt_test.DB.Close() will panic if pages not freed properly. -func TestBucket_DeleteBucket_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - widgets, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - foo, err := widgets.CreateBucket([]byte("foo")) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 1000; i++ { - if err := foo.Put([]byte(fmt.Sprintf("%d", i)), []byte(fmt.Sprintf("%0100d", i))); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - if err := tx.DeleteBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a simple value retrieved via Bucket() returns a nil. -func TestBucket_Bucket_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - widgets, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if err := widgets.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if b := tx.Bucket([]byte("widgets")).Bucket([]byte("foo")); b != nil { - t.Fatal("expected nil bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that creating a bucket on an existing non-bucket key returns an error. -func TestBucket_CreateBucket_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - widgets, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if err := widgets.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if _, err := widgets.CreateBucket([]byte("foo")); err != bolt.ErrIncompatibleValue { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that deleting a bucket on an existing non-bucket key returns an error. -func TestBucket_DeleteBucket_IncompatibleValue(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - widgets, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := widgets.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if err := tx.Bucket([]byte("widgets")).DeleteBucket([]byte("foo")); err != bolt.ErrIncompatibleValue { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can return an autoincrementing sequence. -func TestBucket_NextSequence(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - widgets, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - woojits, err := tx.CreateBucket([]byte("woojits")) - if err != nil { - t.Fatal(err) - } - - // Make sure sequence increments. - if seq, err := widgets.NextSequence(); err != nil { - t.Fatal(err) - } else if seq != 1 { - t.Fatalf("unexpecte sequence: %d", seq) - } - - if seq, err := widgets.NextSequence(); err != nil { - t.Fatal(err) - } else if seq != 2 { - t.Fatalf("unexpected sequence: %d", seq) - } - - // Buckets should be separate. - if seq, err := woojits.NextSequence(); err != nil { - t.Fatal(err) - } else if seq != 1 { - t.Fatalf("unexpected sequence: %d", 1) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket will persist an autoincrementing sequence even if its -// the only thing updated on the bucket. -// https://github.com/boltdb/bolt/issues/296 -func TestBucket_NextSequence_Persist(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.Bucket([]byte("widgets")).NextSequence(); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - seq, err := tx.Bucket([]byte("widgets")).NextSequence() - if err != nil { - t.Fatalf("unexpected error: %s", err) - } else if seq != 2 { - t.Fatalf("unexpected sequence: %d", seq) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that retrieving the next sequence on a read-only bucket returns an error. -func TestBucket_NextSequence_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - _, err := tx.Bucket([]byte("widgets")).NextSequence() - if err != bolt.ErrTxNotWritable { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that retrieving the next sequence for a bucket on a closed database return an error. -func TestBucket_NextSequence_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - if _, err := b.NextSequence(); err != bolt.ErrTxClosed { - t.Fatal(err) - } -} - -// Ensure a user can loop over all key/value pairs in a bucket. -func TestBucket_ForEach(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("0000")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte("0001")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("bar"), []byte("0002")); err != nil { - t.Fatal(err) - } - - var index int - if err := b.ForEach(func(k, v []byte) error { - switch index { - case 0: - if !bytes.Equal(k, []byte("bar")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte("0002")) { - t.Fatalf("unexpected value: %v", v) - } - case 1: - if !bytes.Equal(k, []byte("baz")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte("0001")) { - t.Fatalf("unexpected value: %v", v) - } - case 2: - if !bytes.Equal(k, []byte("foo")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte("0000")) { - t.Fatalf("unexpected value: %v", v) - } - } - index++ - return nil - }); err != nil { - t.Fatal(err) - } - - if index != 3 { - t.Fatalf("unexpected index: %d", index) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a database can stop iteration early. -func TestBucket_ForEach_ShortCircuit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("bar"), []byte("0000")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte("0000")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("0000")); err != nil { - t.Fatal(err) - } - - var index int - if err := tx.Bucket([]byte("widgets")).ForEach(func(k, v []byte) error { - index++ - if bytes.Equal(k, []byte("baz")) { - return errors.New("marker") - } - return nil - }); err == nil || err.Error() != "marker" { - t.Fatalf("unexpected error: %s", err) - } - if index != 2 { - t.Fatalf("unexpected index: %d", index) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that looping over a bucket on a closed database returns an error. -func TestBucket_ForEach_Closed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - - if err := b.ForEach(func(k, v []byte) error { return nil }); err != bolt.ErrTxClosed { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that an error is returned when inserting with an empty key. -func TestBucket_Put_EmptyKey(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte(""), []byte("bar")); err != bolt.ErrKeyRequired { - t.Fatalf("unexpected error: %s", err) - } - if err := b.Put(nil, []byte("bar")); err != bolt.ErrKeyRequired { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that an error is returned when inserting with a key that's too large. -func TestBucket_Put_KeyTooLarge(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put(make([]byte, 32769), []byte("bar")); err != bolt.ErrKeyTooLarge { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that an error is returned when inserting a value that's too large. -func TestBucket_Put_ValueTooLarge(t *testing.T) { - // Skip this test on DroneCI because the machine is resource constrained. - if os.Getenv("DRONE") == "true" { - t.Skip("not enough RAM for test") - } - - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), make([]byte, bolt.MaxValueSize+1)); err != bolt.ErrValueTooLarge { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a bucket can calculate stats. -func TestBucket_Stats(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - // Add bucket with fewer keys but one big value. - bigKey := []byte("really-big-value") - for i := 0; i < 500; i++ { - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte("woojits")) - if err != nil { - t.Fatal(err) - } - - if err := b.Put([]byte(fmt.Sprintf("%03d", i)), []byte(strconv.Itoa(i))); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - } - if err := db.Update(func(tx *bolt.Tx) error { - if err := tx.Bucket([]byte("woojits")).Put(bigKey, []byte(strings.Repeat("*", 10000))); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - db.MustCheck() - - if err := db.View(func(tx *bolt.Tx) error { - stats := tx.Bucket([]byte("woojits")).Stats() - if stats.BranchPageN != 1 { - t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) - } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) - } else if stats.LeafPageN != 7 { - t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) - } else if stats.LeafOverflowN != 2 { - t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) - } else if stats.KeyN != 501 { - t.Fatalf("unexpected KeyN: %d", stats.KeyN) - } else if stats.Depth != 2 { - t.Fatalf("unexpected Depth: %d", stats.Depth) - } - - branchInuse := 16 // branch page header - branchInuse += 7 * 16 // branch elements - branchInuse += 7 * 3 // branch keys (6 3-byte keys) - if stats.BranchInuse != branchInuse { - t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) - } - - leafInuse := 7 * 16 // leaf page header - leafInuse += 501 * 16 // leaf elements - leafInuse += 500*3 + len(bigKey) // leaf keys - leafInuse += 1*10 + 2*90 + 3*400 + 10000 // leaf values - if stats.LeafInuse != leafInuse { - t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) - } - - // Only check allocations for 4KB pages. - if os.Getpagesize() == 4096 { - if stats.BranchAlloc != 4096 { - t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) - } else if stats.LeafAlloc != 36864 { - t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) - } - } - - if stats.BucketN != 1 { - t.Fatalf("unexpected BucketN: %d", stats.BucketN) - } else if stats.InlineBucketN != 0 { - t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) - } else if stats.InlineBucketInuse != 0 { - t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a bucket with random insertion utilizes fill percentage correctly. -func TestBucket_Stats_RandomFill(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } else if os.Getpagesize() != 4096 { - t.Skip("invalid page size for test") - } - - db := MustOpenDB() - defer db.MustClose() - - // Add a set of values in random order. It will be the same random - // order so we can maintain consistency between test runs. - var count int - rand := rand.New(rand.NewSource(42)) - for _, i := range rand.Perm(1000) { - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte("woojits")) - if err != nil { - t.Fatal(err) - } - b.FillPercent = 0.9 - for _, j := range rand.Perm(100) { - index := (j * 10000) + i - if err := b.Put([]byte(fmt.Sprintf("%d000000000000000", index)), []byte("0000000000")); err != nil { - t.Fatal(err) - } - count++ - } - return nil - }); err != nil { - t.Fatal(err) - } - } - - db.MustCheck() - - if err := db.View(func(tx *bolt.Tx) error { - stats := tx.Bucket([]byte("woojits")).Stats() - if stats.KeyN != 100000 { - t.Fatalf("unexpected KeyN: %d", stats.KeyN) - } - - if stats.BranchPageN != 98 { - t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) - } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) - } else if stats.BranchInuse != 130984 { - t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) - } else if stats.BranchAlloc != 401408 { - t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) - } - - if stats.LeafPageN != 3412 { - t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) - } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) - } else if stats.LeafInuse != 4742482 { - t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) - } else if stats.LeafAlloc != 13975552 { - t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a bucket can calculate stats. -func TestBucket_Stats_Small(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - // Add a bucket that fits on a single root leaf. - b, err := tx.CreateBucket([]byte("whozawhats")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - - return nil - }); err != nil { - t.Fatal(err) - } - - db.MustCheck() - - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("whozawhats")) - stats := b.Stats() - if stats.BranchPageN != 0 { - t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) - } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) - } else if stats.LeafPageN != 0 { - t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) - } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) - } else if stats.KeyN != 1 { - t.Fatalf("unexpected KeyN: %d", stats.KeyN) - } else if stats.Depth != 1 { - t.Fatalf("unexpected Depth: %d", stats.Depth) - } else if stats.BranchInuse != 0 { - t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) - } else if stats.LeafInuse != 0 { - t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) - } - - if os.Getpagesize() == 4096 { - if stats.BranchAlloc != 0 { - t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) - } else if stats.LeafAlloc != 0 { - t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) - } - } - - if stats.BucketN != 1 { - t.Fatalf("unexpected BucketN: %d", stats.BucketN) - } else if stats.InlineBucketN != 1 { - t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) - } else if stats.InlineBucketInuse != 16+16+6 { - t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -func TestBucket_Stats_EmptyBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - // Add a bucket that fits on a single root leaf. - if _, err := tx.CreateBucket([]byte("whozawhats")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - db.MustCheck() - - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("whozawhats")) - stats := b.Stats() - if stats.BranchPageN != 0 { - t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) - } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) - } else if stats.LeafPageN != 0 { - t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) - } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) - } else if stats.KeyN != 0 { - t.Fatalf("unexpected KeyN: %d", stats.KeyN) - } else if stats.Depth != 1 { - t.Fatalf("unexpected Depth: %d", stats.Depth) - } else if stats.BranchInuse != 0 { - t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) - } else if stats.LeafInuse != 0 { - t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) - } - - if os.Getpagesize() == 4096 { - if stats.BranchAlloc != 0 { - t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) - } else if stats.LeafAlloc != 0 { - t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) - } - } - - if stats.BucketN != 1 { - t.Fatalf("unexpected BucketN: %d", stats.BucketN) - } else if stats.InlineBucketN != 1 { - t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) - } else if stats.InlineBucketInuse != 16 { - t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a bucket can calculate stats. -func TestBucket_Stats_Nested(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("foo")) - if err != nil { - t.Fatal(err) - } - for i := 0; i < 100; i++ { - if err := b.Put([]byte(fmt.Sprintf("%02d", i)), []byte(fmt.Sprintf("%02d", i))); err != nil { - t.Fatal(err) - } - } - - bar, err := b.CreateBucket([]byte("bar")) - if err != nil { - t.Fatal(err) - } - for i := 0; i < 10; i++ { - if err := bar.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { - t.Fatal(err) - } - } - - baz, err := bar.CreateBucket([]byte("baz")) - if err != nil { - t.Fatal(err) - } - for i := 0; i < 10; i++ { - if err := baz.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { - t.Fatal(err) - } - } - - return nil - }); err != nil { - t.Fatal(err) - } - - db.MustCheck() - - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("foo")) - stats := b.Stats() - if stats.BranchPageN != 0 { - t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) - } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) - } else if stats.LeafPageN != 2 { - t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) - } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) - } else if stats.KeyN != 122 { - t.Fatalf("unexpected KeyN: %d", stats.KeyN) - } else if stats.Depth != 3 { - t.Fatalf("unexpected Depth: %d", stats.Depth) - } else if stats.BranchInuse != 0 { - t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) - } - - foo := 16 // foo (pghdr) - foo += 101 * 16 // foo leaf elements - foo += 100*2 + 100*2 // foo leaf key/values - foo += 3 + 16 // foo -> bar key/value - - bar := 16 // bar (pghdr) - bar += 11 * 16 // bar leaf elements - bar += 10 + 10 // bar leaf key/values - bar += 3 + 16 // bar -> baz key/value - - baz := 16 // baz (inline) (pghdr) - baz += 10 * 16 // baz leaf elements - baz += 10 + 10 // baz leaf key/values - - if stats.LeafInuse != foo+bar+baz { - t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) - } - - if os.Getpagesize() == 4096 { - if stats.BranchAlloc != 0 { - t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) - } else if stats.LeafAlloc != 8192 { - t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) - } - } - - if stats.BucketN != 3 { - t.Fatalf("unexpected BucketN: %d", stats.BucketN) - } else if stats.InlineBucketN != 1 { - t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) - } else if stats.InlineBucketInuse != baz { - t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a large bucket can calculate stats. -func TestBucket_Stats_Large(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - db := MustOpenDB() - defer db.MustClose() - - var index int - for i := 0; i < 100; i++ { - // Add bucket with lots of keys. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - for i := 0; i < 1000; i++ { - if err := b.Put([]byte(strconv.Itoa(index)), []byte(strconv.Itoa(index))); err != nil { - t.Fatal(err) - } - index++ - } - return nil - }); err != nil { - t.Fatal(err) - } - } - - db.MustCheck() - - if err := db.View(func(tx *bolt.Tx) error { - stats := tx.Bucket([]byte("widgets")).Stats() - if stats.BranchPageN != 13 { - t.Fatalf("unexpected BranchPageN: %d", stats.BranchPageN) - } else if stats.BranchOverflowN != 0 { - t.Fatalf("unexpected BranchOverflowN: %d", stats.BranchOverflowN) - } else if stats.LeafPageN != 1196 { - t.Fatalf("unexpected LeafPageN: %d", stats.LeafPageN) - } else if stats.LeafOverflowN != 0 { - t.Fatalf("unexpected LeafOverflowN: %d", stats.LeafOverflowN) - } else if stats.KeyN != 100000 { - t.Fatalf("unexpected KeyN: %d", stats.KeyN) - } else if stats.Depth != 3 { - t.Fatalf("unexpected Depth: %d", stats.Depth) - } else if stats.BranchInuse != 25257 { - t.Fatalf("unexpected BranchInuse: %d", stats.BranchInuse) - } else if stats.LeafInuse != 2596916 { - t.Fatalf("unexpected LeafInuse: %d", stats.LeafInuse) - } - - if os.Getpagesize() == 4096 { - if stats.BranchAlloc != 53248 { - t.Fatalf("unexpected BranchAlloc: %d", stats.BranchAlloc) - } else if stats.LeafAlloc != 4898816 { - t.Fatalf("unexpected LeafAlloc: %d", stats.LeafAlloc) - } - } - - if stats.BucketN != 1 { - t.Fatalf("unexpected BucketN: %d", stats.BucketN) - } else if stats.InlineBucketN != 0 { - t.Fatalf("unexpected InlineBucketN: %d", stats.InlineBucketN) - } else if stats.InlineBucketInuse != 0 { - t.Fatalf("unexpected InlineBucketInuse: %d", stats.InlineBucketInuse) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can write random keys and values across multiple transactions. -func TestBucket_Put_Single(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - index := 0 - if err := quick.Check(func(items testdata) bool { - db := MustOpenDB() - defer db.MustClose() - - m := make(map[string][]byte) - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - for _, item := range items { - if err := db.Update(func(tx *bolt.Tx) error { - if err := tx.Bucket([]byte("widgets")).Put(item.Key, item.Value); err != nil { - panic("put error: " + err.Error()) - } - m[string(item.Key)] = item.Value - return nil - }); err != nil { - t.Fatal(err) - } - - // Verify all key/values so far. - if err := db.View(func(tx *bolt.Tx) error { - i := 0 - for k, v := range m { - value := tx.Bucket([]byte("widgets")).Get([]byte(k)) - if !bytes.Equal(value, v) { - t.Logf("value mismatch [run %d] (%d of %d):\nkey: %x\ngot: %x\nexp: %x", index, i, len(m), []byte(k), value, v) - db.CopyTempFile() - t.FailNow() - } - i++ - } - return nil - }); err != nil { - t.Fatal(err) - } - } - - index++ - return true - }, nil); err != nil { - t.Error(err) - } -} - -// Ensure that a transaction can insert multiple key/value pairs at once. -func TestBucket_Put_Multiple(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - if err := quick.Check(func(items testdata) bool { - db := MustOpenDB() - defer db.MustClose() - - // Bulk insert all values. - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for _, item := range items { - if err := b.Put(item.Key, item.Value); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Verify all items exist. - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for _, item := range items { - value := b.Get(item.Key) - if !bytes.Equal(item.Value, value) { - db.CopyTempFile() - t.Fatalf("exp=%x; got=%x", item.Value, value) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - return true - }, qconfig()); err != nil { - t.Error(err) - } -} - -// Ensure that a transaction can delete all key/value pairs and return to a single leaf page. -func TestBucket_Delete_Quick(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - if err := quick.Check(func(items testdata) bool { - db := MustOpenDB() - defer db.MustClose() - - // Bulk insert all values. - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for _, item := range items { - if err := b.Put(item.Key, item.Value); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Remove items one at a time and check consistency. - for _, item := range items { - if err := db.Update(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("widgets")).Delete(item.Key) - }); err != nil { - t.Fatal(err) - } - } - - // Anything before our deletion index should be nil. - if err := db.View(func(tx *bolt.Tx) error { - if err := tx.Bucket([]byte("widgets")).ForEach(func(k, v []byte) error { - t.Fatalf("bucket should be empty; found: %06x", trunc(k, 3)) - return nil - }); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - return true - }, qconfig()); err != nil { - t.Error(err) - } -} - -func ExampleBucket_Put() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Start a write transaction. - if err := db.Update(func(tx *bolt.Tx) error { - // Create a bucket. - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - return err - } - - // Set the value "bar" for the key "foo". - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - return err - } - return nil - }); err != nil { - log.Fatal(err) - } - - // Read value back in a different read-only transaction. - if err := db.View(func(tx *bolt.Tx) error { - value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value of 'foo' is: %s\n", value) - return nil - }); err != nil { - log.Fatal(err) - } - - // Close database to release file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // The value of 'foo' is: bar -} - -func ExampleBucket_Delete() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Start a write transaction. - if err := db.Update(func(tx *bolt.Tx) error { - // Create a bucket. - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - return err - } - - // Set the value "bar" for the key "foo". - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - return err - } - - // Retrieve the key back from the database and verify it. - value := b.Get([]byte("foo")) - fmt.Printf("The value of 'foo' was: %s\n", value) - - return nil - }); err != nil { - log.Fatal(err) - } - - // Delete the key in a different write transaction. - if err := db.Update(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("widgets")).Delete([]byte("foo")) - }); err != nil { - log.Fatal(err) - } - - // Retrieve the key again. - if err := db.View(func(tx *bolt.Tx) error { - value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - if value == nil { - fmt.Printf("The value of 'foo' is now: nil\n") - } - return nil - }); err != nil { - log.Fatal(err) - } - - // Close database to release file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // The value of 'foo' was: bar - // The value of 'foo' is now: nil -} - -func ExampleBucket_ForEach() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Insert data into a bucket. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("animals")) - if err != nil { - return err - } - - if err := b.Put([]byte("dog"), []byte("fun")); err != nil { - return err - } - if err := b.Put([]byte("cat"), []byte("lame")); err != nil { - return err - } - if err := b.Put([]byte("liger"), []byte("awesome")); err != nil { - return err - } - - // Iterate over items in sorted key order. - if err := b.ForEach(func(k, v []byte) error { - fmt.Printf("A %s is %s.\n", k, v) - return nil - }); err != nil { - return err - } - - return nil - }); err != nil { - log.Fatal(err) - } - - // Close database to release file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // A cat is lame. - // A dog is fun. - // A liger is awesome. -} diff --git a/vendor/github.com/boltdb/bolt/cmd/bolt/main.go b/vendor/github.com/boltdb/bolt/cmd/bolt/main.go deleted file mode 100644 index b96e6f73..00000000 --- a/vendor/github.com/boltdb/bolt/cmd/bolt/main.go +++ /dev/null @@ -1,1532 +0,0 @@ -package main - -import ( - "bytes" - "encoding/binary" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "math/rand" - "os" - "runtime" - "runtime/pprof" - "strconv" - "strings" - "time" - "unicode" - "unicode/utf8" - "unsafe" - - "github.com/boltdb/bolt" -) - -var ( - // ErrUsage is returned when a usage message was printed and the process - // should simply exit with an error. - ErrUsage = errors.New("usage") - - // ErrUnknownCommand is returned when a CLI command is not specified. - ErrUnknownCommand = errors.New("unknown command") - - // ErrPathRequired is returned when the path to a Bolt database is not specified. - ErrPathRequired = errors.New("path required") - - // ErrFileNotFound is returned when a Bolt database does not exist. - ErrFileNotFound = errors.New("file not found") - - // ErrInvalidValue is returned when a benchmark reads an unexpected value. - ErrInvalidValue = errors.New("invalid value") - - // ErrCorrupt is returned when a checking a data file finds errors. - ErrCorrupt = errors.New("invalid value") - - // ErrNonDivisibleBatchSize is returned when the batch size can't be evenly - // divided by the iteration count. - ErrNonDivisibleBatchSize = errors.New("number of iterations must be divisible by the batch size") - - // ErrPageIDRequired is returned when a required page id is not specified. - ErrPageIDRequired = errors.New("page id required") - - // ErrPageNotFound is returned when specifying a page above the high water mark. - ErrPageNotFound = errors.New("page not found") - - // ErrPageFreed is returned when reading a page that has already been freed. - ErrPageFreed = errors.New("page freed") -) - -// PageHeaderSize represents the size of the bolt.page header. -const PageHeaderSize = 16 - -func main() { - m := NewMain() - if err := m.Run(os.Args[1:]...); err == ErrUsage { - os.Exit(2) - } else if err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } -} - -// Main represents the main program execution. -type Main struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewMain returns a new instance of Main connect to the standard input/output. -func NewMain() *Main { - return &Main{ - Stdin: os.Stdin, - Stdout: os.Stdout, - Stderr: os.Stderr, - } -} - -// Run executes the program. -func (m *Main) Run(args ...string) error { - // Require a command at the beginning. - if len(args) == 0 || strings.HasPrefix(args[0], "-") { - fmt.Fprintln(m.Stderr, m.Usage()) - return ErrUsage - } - - // Execute command. - switch args[0] { - case "help": - fmt.Fprintln(m.Stderr, m.Usage()) - return ErrUsage - case "bench": - return newBenchCommand(m).Run(args[1:]...) - case "check": - return newCheckCommand(m).Run(args[1:]...) - case "dump": - return newDumpCommand(m).Run(args[1:]...) - case "info": - return newInfoCommand(m).Run(args[1:]...) - case "page": - return newPageCommand(m).Run(args[1:]...) - case "pages": - return newPagesCommand(m).Run(args[1:]...) - case "stats": - return newStatsCommand(m).Run(args[1:]...) - default: - return ErrUnknownCommand - } -} - -// Usage returns the help message. -func (m *Main) Usage() string { - return strings.TrimLeft(` -Bolt is a tool for inspecting bolt databases. - -Usage: - - bolt command [arguments] - -The commands are: - - bench run synthetic benchmark against bolt - check verifies integrity of bolt database - info print basic info - help print this screen - pages print list of pages with their types - stats iterate over all pages and generate usage stats - -Use "bolt [command] -h" for more information about a command. -`, "\n") -} - -// CheckCommand represents the "check" command execution. -type CheckCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewCheckCommand returns a CheckCommand. -func newCheckCommand(m *Main) *CheckCommand { - return &CheckCommand{ - Stdin: m.Stdin, - Stdout: m.Stdout, - Stderr: m.Stderr, - } -} - -// Run executes the command. -func (cmd *CheckCommand) Run(args ...string) error { - // Parse flags. - fs := flag.NewFlagSet("", flag.ContinueOnError) - help := fs.Bool("h", false, "") - if err := fs.Parse(args); err != nil { - return err - } else if *help { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - // Require database path. - path := fs.Arg(0) - if path == "" { - return ErrPathRequired - } else if _, err := os.Stat(path); os.IsNotExist(err) { - return ErrFileNotFound - } - - // Open database. - db, err := bolt.Open(path, 0666, nil) - if err != nil { - return err - } - defer db.Close() - - // Perform consistency check. - return db.View(func(tx *bolt.Tx) error { - var count int - ch := tx.Check() - loop: - for { - select { - case err, ok := <-ch: - if !ok { - break loop - } - fmt.Fprintln(cmd.Stdout, err) - count++ - } - } - - // Print summary of errors. - if count > 0 { - fmt.Fprintf(cmd.Stdout, "%d errors found\n", count) - return ErrCorrupt - } - - // Notify user that database is valid. - fmt.Fprintln(cmd.Stdout, "OK") - return nil - }) -} - -// Usage returns the help message. -func (cmd *CheckCommand) Usage() string { - return strings.TrimLeft(` -usage: bolt check PATH - -Check opens a database at PATH and runs an exhaustive check to verify that -all pages are accessible or are marked as freed. It also verifies that no -pages are double referenced. - -Verification errors will stream out as they are found and the process will -return after all pages have been checked. -`, "\n") -} - -// InfoCommand represents the "info" command execution. -type InfoCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewInfoCommand returns a InfoCommand. -func newInfoCommand(m *Main) *InfoCommand { - return &InfoCommand{ - Stdin: m.Stdin, - Stdout: m.Stdout, - Stderr: m.Stderr, - } -} - -// Run executes the command. -func (cmd *InfoCommand) Run(args ...string) error { - // Parse flags. - fs := flag.NewFlagSet("", flag.ContinueOnError) - help := fs.Bool("h", false, "") - if err := fs.Parse(args); err != nil { - return err - } else if *help { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - // Require database path. - path := fs.Arg(0) - if path == "" { - return ErrPathRequired - } else if _, err := os.Stat(path); os.IsNotExist(err) { - return ErrFileNotFound - } - - // Open the database. - db, err := bolt.Open(path, 0666, nil) - if err != nil { - return err - } - defer db.Close() - - // Print basic database info. - info := db.Info() - fmt.Fprintf(cmd.Stdout, "Page Size: %d\n", info.PageSize) - - return nil -} - -// Usage returns the help message. -func (cmd *InfoCommand) Usage() string { - return strings.TrimLeft(` -usage: bolt info PATH - -Info prints basic information about the Bolt database at PATH. -`, "\n") -} - -// DumpCommand represents the "dump" command execution. -type DumpCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// newDumpCommand returns a DumpCommand. -func newDumpCommand(m *Main) *DumpCommand { - return &DumpCommand{ - Stdin: m.Stdin, - Stdout: m.Stdout, - Stderr: m.Stderr, - } -} - -// Run executes the command. -func (cmd *DumpCommand) Run(args ...string) error { - // Parse flags. - fs := flag.NewFlagSet("", flag.ContinueOnError) - help := fs.Bool("h", false, "") - if err := fs.Parse(args); err != nil { - return err - } else if *help { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - // Require database path and page id. - path := fs.Arg(0) - if path == "" { - return ErrPathRequired - } else if _, err := os.Stat(path); os.IsNotExist(err) { - return ErrFileNotFound - } - - // Read page ids. - pageIDs, err := atois(fs.Args()[1:]) - if err != nil { - return err - } else if len(pageIDs) == 0 { - return ErrPageIDRequired - } - - // Open database to retrieve page size. - pageSize, err := ReadPageSize(path) - if err != nil { - return err - } - - // Open database file handler. - f, err := os.Open(path) - if err != nil { - return err - } - defer func() { _ = f.Close() }() - - // Print each page listed. - for i, pageID := range pageIDs { - // Print a separator. - if i > 0 { - fmt.Fprintln(cmd.Stdout, "===============================================") - } - - // Print page to stdout. - if err := cmd.PrintPage(cmd.Stdout, f, pageID, pageSize); err != nil { - return err - } - } - - return nil -} - -// PrintPage prints a given page as hexidecimal. -func (cmd *DumpCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { - const bytesPerLineN = 16 - - // Read page into buffer. - buf := make([]byte, pageSize) - addr := pageID * pageSize - if n, err := r.ReadAt(buf, int64(addr)); err != nil { - return err - } else if n != pageSize { - return io.ErrUnexpectedEOF - } - - // Write out to writer in 16-byte lines. - var prev []byte - var skipped bool - for offset := 0; offset < pageSize; offset += bytesPerLineN { - // Retrieve current 16-byte line. - line := buf[offset : offset+bytesPerLineN] - isLastLine := (offset == (pageSize - bytesPerLineN)) - - // If it's the same as the previous line then print a skip. - if bytes.Equal(line, prev) && !isLastLine { - if !skipped { - fmt.Fprintf(w, "%07x *\n", addr+offset) - skipped = true - } - } else { - // Print line as hexadecimal in 2-byte groups. - fmt.Fprintf(w, "%07x %04x %04x %04x %04x %04x %04x %04x %04x\n", addr+offset, - line[0:2], line[2:4], line[4:6], line[6:8], - line[8:10], line[10:12], line[12:14], line[14:16], - ) - - skipped = false - } - - // Save the previous line. - prev = line - } - fmt.Fprint(w, "\n") - - return nil -} - -// Usage returns the help message. -func (cmd *DumpCommand) Usage() string { - return strings.TrimLeft(` -usage: bolt dump -page PAGEID PATH - -Dump prints a hexidecimal dump of a single page. -`, "\n") -} - -// PageCommand represents the "page" command execution. -type PageCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// newPageCommand returns a PageCommand. -func newPageCommand(m *Main) *PageCommand { - return &PageCommand{ - Stdin: m.Stdin, - Stdout: m.Stdout, - Stderr: m.Stderr, - } -} - -// Run executes the command. -func (cmd *PageCommand) Run(args ...string) error { - // Parse flags. - fs := flag.NewFlagSet("", flag.ContinueOnError) - help := fs.Bool("h", false, "") - if err := fs.Parse(args); err != nil { - return err - } else if *help { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - // Require database path and page id. - path := fs.Arg(0) - if path == "" { - return ErrPathRequired - } else if _, err := os.Stat(path); os.IsNotExist(err) { - return ErrFileNotFound - } - - // Read page ids. - pageIDs, err := atois(fs.Args()[1:]) - if err != nil { - return err - } else if len(pageIDs) == 0 { - return ErrPageIDRequired - } - - // Open database file handler. - f, err := os.Open(path) - if err != nil { - return err - } - defer func() { _ = f.Close() }() - - // Print each page listed. - for i, pageID := range pageIDs { - // Print a separator. - if i > 0 { - fmt.Fprintln(cmd.Stdout, "===============================================") - } - - // Retrieve page info and page size. - p, buf, err := ReadPage(path, pageID) - if err != nil { - return err - } - - // Print basic page info. - fmt.Fprintf(cmd.Stdout, "Page ID: %d\n", p.id) - fmt.Fprintf(cmd.Stdout, "Page Type: %s\n", p.Type()) - fmt.Fprintf(cmd.Stdout, "Total Size: %d bytes\n", len(buf)) - - // Print type-specific data. - switch p.Type() { - case "meta": - err = cmd.PrintMeta(cmd.Stdout, buf) - case "leaf": - err = cmd.PrintLeaf(cmd.Stdout, buf) - case "branch": - err = cmd.PrintBranch(cmd.Stdout, buf) - case "freelist": - err = cmd.PrintFreelist(cmd.Stdout, buf) - } - if err != nil { - return err - } - } - - return nil -} - -// PrintMeta prints the data from the meta page. -func (cmd *PageCommand) PrintMeta(w io.Writer, buf []byte) error { - m := (*meta)(unsafe.Pointer(&buf[PageHeaderSize])) - fmt.Fprintf(w, "Version: %d\n", m.version) - fmt.Fprintf(w, "Page Size: %d bytes\n", m.pageSize) - fmt.Fprintf(w, "Flags: %08x\n", m.flags) - fmt.Fprintf(w, "Root: \n", m.root.root) - fmt.Fprintf(w, "Freelist: \n", m.freelist) - fmt.Fprintf(w, "HWM: \n", m.pgid) - fmt.Fprintf(w, "Txn ID: %d\n", m.txid) - fmt.Fprintf(w, "Checksum: %016x\n", m.checksum) - fmt.Fprintf(w, "\n") - return nil -} - -// PrintLeaf prints the data for a leaf page. -func (cmd *PageCommand) PrintLeaf(w io.Writer, buf []byte) error { - p := (*page)(unsafe.Pointer(&buf[0])) - - // Print number of items. - fmt.Fprintf(w, "Item Count: %d\n", p.count) - fmt.Fprintf(w, "\n") - - // Print each key/value. - for i := uint16(0); i < p.count; i++ { - e := p.leafPageElement(i) - - // Format key as string. - var k string - if isPrintable(string(e.key())) { - k = fmt.Sprintf("%q", string(e.key())) - } else { - k = fmt.Sprintf("%x", string(e.key())) - } - - // Format value as string. - var v string - if (e.flags & uint32(bucketLeafFlag)) != 0 { - b := (*bucket)(unsafe.Pointer(&e.value()[0])) - v = fmt.Sprintf("", b.root, b.sequence) - } else if isPrintable(string(e.value())) { - k = fmt.Sprintf("%q", string(e.value())) - } else { - k = fmt.Sprintf("%x", string(e.value())) - } - - fmt.Fprintf(w, "%s: %s\n", k, v) - } - fmt.Fprintf(w, "\n") - return nil -} - -// PrintBranch prints the data for a leaf page. -func (cmd *PageCommand) PrintBranch(w io.Writer, buf []byte) error { - p := (*page)(unsafe.Pointer(&buf[0])) - - // Print number of items. - fmt.Fprintf(w, "Item Count: %d\n", p.count) - fmt.Fprintf(w, "\n") - - // Print each key/value. - for i := uint16(0); i < p.count; i++ { - e := p.branchPageElement(i) - - // Format key as string. - var k string - if isPrintable(string(e.key())) { - k = fmt.Sprintf("%q", string(e.key())) - } else { - k = fmt.Sprintf("%x", string(e.key())) - } - - fmt.Fprintf(w, "%s: \n", k, e.pgid) - } - fmt.Fprintf(w, "\n") - return nil -} - -// PrintFreelist prints the data for a freelist page. -func (cmd *PageCommand) PrintFreelist(w io.Writer, buf []byte) error { - p := (*page)(unsafe.Pointer(&buf[0])) - - // Print number of items. - fmt.Fprintf(w, "Item Count: %d\n", p.count) - fmt.Fprintf(w, "\n") - - // Print each page in the freelist. - ids := (*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)) - for i := uint16(0); i < p.count; i++ { - fmt.Fprintf(w, "%d\n", ids[i]) - } - fmt.Fprintf(w, "\n") - return nil -} - -// PrintPage prints a given page as hexidecimal. -func (cmd *PageCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { - const bytesPerLineN = 16 - - // Read page into buffer. - buf := make([]byte, pageSize) - addr := pageID * pageSize - if n, err := r.ReadAt(buf, int64(addr)); err != nil { - return err - } else if n != pageSize { - return io.ErrUnexpectedEOF - } - - // Write out to writer in 16-byte lines. - var prev []byte - var skipped bool - for offset := 0; offset < pageSize; offset += bytesPerLineN { - // Retrieve current 16-byte line. - line := buf[offset : offset+bytesPerLineN] - isLastLine := (offset == (pageSize - bytesPerLineN)) - - // If it's the same as the previous line then print a skip. - if bytes.Equal(line, prev) && !isLastLine { - if !skipped { - fmt.Fprintf(w, "%07x *\n", addr+offset) - skipped = true - } - } else { - // Print line as hexadecimal in 2-byte groups. - fmt.Fprintf(w, "%07x %04x %04x %04x %04x %04x %04x %04x %04x\n", addr+offset, - line[0:2], line[2:4], line[4:6], line[6:8], - line[8:10], line[10:12], line[12:14], line[14:16], - ) - - skipped = false - } - - // Save the previous line. - prev = line - } - fmt.Fprint(w, "\n") - - return nil -} - -// Usage returns the help message. -func (cmd *PageCommand) Usage() string { - return strings.TrimLeft(` -usage: bolt page -page PATH pageid [pageid...] - -Page prints one or more pages in human readable format. -`, "\n") -} - -// PagesCommand represents the "pages" command execution. -type PagesCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewPagesCommand returns a PagesCommand. -func newPagesCommand(m *Main) *PagesCommand { - return &PagesCommand{ - Stdin: m.Stdin, - Stdout: m.Stdout, - Stderr: m.Stderr, - } -} - -// Run executes the command. -func (cmd *PagesCommand) Run(args ...string) error { - // Parse flags. - fs := flag.NewFlagSet("", flag.ContinueOnError) - help := fs.Bool("h", false, "") - if err := fs.Parse(args); err != nil { - return err - } else if *help { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - // Require database path. - path := fs.Arg(0) - if path == "" { - return ErrPathRequired - } else if _, err := os.Stat(path); os.IsNotExist(err) { - return ErrFileNotFound - } - - // Open database. - db, err := bolt.Open(path, 0666, nil) - if err != nil { - return err - } - defer func() { _ = db.Close() }() - - // Write header. - fmt.Fprintln(cmd.Stdout, "ID TYPE ITEMS OVRFLW") - fmt.Fprintln(cmd.Stdout, "======== ========== ====== ======") - - return db.Update(func(tx *bolt.Tx) error { - var id int - for { - p, err := tx.Page(id) - if err != nil { - return &PageError{ID: id, Err: err} - } else if p == nil { - break - } - - // Only display count and overflow if this is a non-free page. - var count, overflow string - if p.Type != "free" { - count = strconv.Itoa(p.Count) - if p.OverflowCount > 0 { - overflow = strconv.Itoa(p.OverflowCount) - } - } - - // Print table row. - fmt.Fprintf(cmd.Stdout, "%-8d %-10s %-6s %-6s\n", p.ID, p.Type, count, overflow) - - // Move to the next non-overflow page. - id += 1 - if p.Type != "free" { - id += p.OverflowCount - } - } - return nil - }) -} - -// Usage returns the help message. -func (cmd *PagesCommand) Usage() string { - return strings.TrimLeft(` -usage: bolt pages PATH - -Pages prints a table of pages with their type (meta, leaf, branch, freelist). -Leaf and branch pages will show a key count in the "items" column while the -freelist will show the number of free pages in the "items" column. - -The "overflow" column shows the number of blocks that the page spills over -into. Normally there is no overflow but large keys and values can cause -a single page to take up multiple blocks. -`, "\n") -} - -// StatsCommand represents the "stats" command execution. -type StatsCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewStatsCommand returns a StatsCommand. -func newStatsCommand(m *Main) *StatsCommand { - return &StatsCommand{ - Stdin: m.Stdin, - Stdout: m.Stdout, - Stderr: m.Stderr, - } -} - -// Run executes the command. -func (cmd *StatsCommand) Run(args ...string) error { - // Parse flags. - fs := flag.NewFlagSet("", flag.ContinueOnError) - help := fs.Bool("h", false, "") - if err := fs.Parse(args); err != nil { - return err - } else if *help { - fmt.Fprintln(cmd.Stderr, cmd.Usage()) - return ErrUsage - } - - // Require database path. - path, prefix := fs.Arg(0), fs.Arg(1) - if path == "" { - return ErrPathRequired - } else if _, err := os.Stat(path); os.IsNotExist(err) { - return ErrFileNotFound - } - - // Open database. - db, err := bolt.Open(path, 0666, nil) - if err != nil { - return err - } - defer db.Close() - - return db.View(func(tx *bolt.Tx) error { - var s bolt.BucketStats - var count int - if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { - if bytes.HasPrefix(name, []byte(prefix)) { - s.Add(b.Stats()) - count += 1 - } - return nil - }); err != nil { - return err - } - - fmt.Fprintf(cmd.Stdout, "Aggregate statistics for %d buckets\n\n", count) - - fmt.Fprintln(cmd.Stdout, "Page count statistics") - fmt.Fprintf(cmd.Stdout, "\tNumber of logical branch pages: %d\n", s.BranchPageN) - fmt.Fprintf(cmd.Stdout, "\tNumber of physical branch overflow pages: %d\n", s.BranchOverflowN) - fmt.Fprintf(cmd.Stdout, "\tNumber of logical leaf pages: %d\n", s.LeafPageN) - fmt.Fprintf(cmd.Stdout, "\tNumber of physical leaf overflow pages: %d\n", s.LeafOverflowN) - - fmt.Fprintln(cmd.Stdout, "Tree statistics") - fmt.Fprintf(cmd.Stdout, "\tNumber of keys/value pairs: %d\n", s.KeyN) - fmt.Fprintf(cmd.Stdout, "\tNumber of levels in B+tree: %d\n", s.Depth) - - fmt.Fprintln(cmd.Stdout, "Page size utilization") - fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical branch pages: %d\n", s.BranchAlloc) - var percentage int - if s.BranchAlloc != 0 { - percentage = int(float32(s.BranchInuse) * 100.0 / float32(s.BranchAlloc)) - } - fmt.Fprintf(cmd.Stdout, "\tBytes actually used for branch data: %d (%d%%)\n", s.BranchInuse, percentage) - fmt.Fprintf(cmd.Stdout, "\tBytes allocated for physical leaf pages: %d\n", s.LeafAlloc) - percentage = 0 - if s.LeafAlloc != 0 { - percentage = int(float32(s.LeafInuse) * 100.0 / float32(s.LeafAlloc)) - } - fmt.Fprintf(cmd.Stdout, "\tBytes actually used for leaf data: %d (%d%%)\n", s.LeafInuse, percentage) - - fmt.Fprintln(cmd.Stdout, "Bucket statistics") - fmt.Fprintf(cmd.Stdout, "\tTotal number of buckets: %d\n", s.BucketN) - percentage = 0 - if s.BucketN != 0 { - percentage = int(float32(s.InlineBucketN) * 100.0 / float32(s.BucketN)) - } - fmt.Fprintf(cmd.Stdout, "\tTotal number on inlined buckets: %d (%d%%)\n", s.InlineBucketN, percentage) - percentage = 0 - if s.LeafInuse != 0 { - percentage = int(float32(s.InlineBucketInuse) * 100.0 / float32(s.LeafInuse)) - } - fmt.Fprintf(cmd.Stdout, "\tBytes used for inlined buckets: %d (%d%%)\n", s.InlineBucketInuse, percentage) - - return nil - }) -} - -// Usage returns the help message. -func (cmd *StatsCommand) Usage() string { - return strings.TrimLeft(` -usage: bolt stats PATH - -Stats performs an extensive search of the database to track every page -reference. It starts at the current meta page and recursively iterates -through every accessible bucket. - -The following errors can be reported: - - already freed - The page is referenced more than once in the freelist. - - unreachable unfreed - The page is not referenced by a bucket or in the freelist. - - reachable freed - The page is referenced by a bucket but is also in the freelist. - - out of bounds - A page is referenced that is above the high water mark. - - multiple references - A page is referenced by more than one other page. - - invalid type - The page type is not "meta", "leaf", "branch", or "freelist". - -No errors should occur in your database. However, if for some reason you -experience corruption, please submit a ticket to the Bolt project page: - - https://github.com/boltdb/bolt/issues -`, "\n") -} - -var benchBucketName = []byte("bench") - -// BenchCommand represents the "bench" command execution. -type BenchCommand struct { - Stdin io.Reader - Stdout io.Writer - Stderr io.Writer -} - -// NewBenchCommand returns a BenchCommand using the -func newBenchCommand(m *Main) *BenchCommand { - return &BenchCommand{ - Stdin: m.Stdin, - Stdout: m.Stdout, - Stderr: m.Stderr, - } -} - -// Run executes the "bench" command. -func (cmd *BenchCommand) Run(args ...string) error { - // Parse CLI arguments. - options, err := cmd.ParseFlags(args) - if err != nil { - return err - } - - // Remove path if "-work" is not set. Otherwise keep path. - if options.Work { - fmt.Fprintf(cmd.Stdout, "work: %s\n", options.Path) - } else { - defer os.Remove(options.Path) - } - - // Create database. - db, err := bolt.Open(options.Path, 0666, nil) - if err != nil { - return err - } - db.NoSync = options.NoSync - defer db.Close() - - // Write to the database. - var results BenchResults - if err := cmd.runWrites(db, options, &results); err != nil { - return fmt.Errorf("write: %v", err) - } - - // Read from the database. - if err := cmd.runReads(db, options, &results); err != nil { - return fmt.Errorf("bench: read: %s", err) - } - - // Print results. - fmt.Fprintf(os.Stderr, "# Write\t%v\t(%v/op)\t(%v op/sec)\n", results.WriteDuration, results.WriteOpDuration(), results.WriteOpsPerSecond()) - fmt.Fprintf(os.Stderr, "# Read\t%v\t(%v/op)\t(%v op/sec)\n", results.ReadDuration, results.ReadOpDuration(), results.ReadOpsPerSecond()) - fmt.Fprintln(os.Stderr, "") - return nil -} - -// ParseFlags parses the command line flags. -func (cmd *BenchCommand) ParseFlags(args []string) (*BenchOptions, error) { - var options BenchOptions - - // Parse flagset. - fs := flag.NewFlagSet("", flag.ContinueOnError) - fs.StringVar(&options.ProfileMode, "profile-mode", "rw", "") - fs.StringVar(&options.WriteMode, "write-mode", "seq", "") - fs.StringVar(&options.ReadMode, "read-mode", "seq", "") - fs.IntVar(&options.Iterations, "count", 1000, "") - fs.IntVar(&options.BatchSize, "batch-size", 0, "") - fs.IntVar(&options.KeySize, "key-size", 8, "") - fs.IntVar(&options.ValueSize, "value-size", 32, "") - fs.StringVar(&options.CPUProfile, "cpuprofile", "", "") - fs.StringVar(&options.MemProfile, "memprofile", "", "") - fs.StringVar(&options.BlockProfile, "blockprofile", "", "") - fs.Float64Var(&options.FillPercent, "fill-percent", bolt.DefaultFillPercent, "") - fs.BoolVar(&options.NoSync, "no-sync", false, "") - fs.BoolVar(&options.Work, "work", false, "") - fs.StringVar(&options.Path, "path", "", "") - fs.SetOutput(cmd.Stderr) - if err := fs.Parse(args); err != nil { - return nil, err - } - - // Set batch size to iteration size if not set. - // Require that batch size can be evenly divided by the iteration count. - if options.BatchSize == 0 { - options.BatchSize = options.Iterations - } else if options.Iterations%options.BatchSize != 0 { - return nil, ErrNonDivisibleBatchSize - } - - // Generate temp path if one is not passed in. - if options.Path == "" { - f, err := ioutil.TempFile("", "bolt-bench-") - if err != nil { - return nil, fmt.Errorf("temp file: %s", err) - } - f.Close() - os.Remove(f.Name()) - options.Path = f.Name() - } - - return &options, nil -} - -// Writes to the database. -func (cmd *BenchCommand) runWrites(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - // Start profiling for writes. - if options.ProfileMode == "rw" || options.ProfileMode == "w" { - cmd.startProfiling(options) - } - - t := time.Now() - - var err error - switch options.WriteMode { - case "seq": - err = cmd.runWritesSequential(db, options, results) - case "rnd": - err = cmd.runWritesRandom(db, options, results) - case "seq-nest": - err = cmd.runWritesSequentialNested(db, options, results) - case "rnd-nest": - err = cmd.runWritesRandomNested(db, options, results) - default: - return fmt.Errorf("invalid write mode: %s", options.WriteMode) - } - - // Save time to write. - results.WriteDuration = time.Since(t) - - // Stop profiling for writes only. - if options.ProfileMode == "w" { - cmd.stopProfiling() - } - - return err -} - -func (cmd *BenchCommand) runWritesSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var i = uint32(0) - return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) -} - -func (cmd *BenchCommand) runWritesRandom(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) -} - -func (cmd *BenchCommand) runWritesSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - var i = uint32(0) - return cmd.runWritesWithSource(db, options, results, func() uint32 { i++; return i }) -} - -func (cmd *BenchCommand) runWritesRandomNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - return cmd.runWritesWithSource(db, options, results, func() uint32 { return r.Uint32() }) -} - -func (cmd *BenchCommand) runWritesWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { - results.WriteOps = options.Iterations - - for i := 0; i < options.Iterations; i += options.BatchSize { - if err := db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists(benchBucketName) - b.FillPercent = options.FillPercent - - for j := 0; j < options.BatchSize; j++ { - key := make([]byte, options.KeySize) - value := make([]byte, options.ValueSize) - - // Write key as uint32. - binary.BigEndian.PutUint32(key, keySource()) - - // Insert key/value. - if err := b.Put(key, value); err != nil { - return err - } - } - - return nil - }); err != nil { - return err - } - } - return nil -} - -func (cmd *BenchCommand) runWritesNestedWithSource(db *bolt.DB, options *BenchOptions, results *BenchResults, keySource func() uint32) error { - results.WriteOps = options.Iterations - - for i := 0; i < options.Iterations; i += options.BatchSize { - if err := db.Update(func(tx *bolt.Tx) error { - top, err := tx.CreateBucketIfNotExists(benchBucketName) - if err != nil { - return err - } - top.FillPercent = options.FillPercent - - // Create bucket key. - name := make([]byte, options.KeySize) - binary.BigEndian.PutUint32(name, keySource()) - - // Create bucket. - b, err := top.CreateBucketIfNotExists(name) - if err != nil { - return err - } - b.FillPercent = options.FillPercent - - for j := 0; j < options.BatchSize; j++ { - var key = make([]byte, options.KeySize) - var value = make([]byte, options.ValueSize) - - // Generate key as uint32. - binary.BigEndian.PutUint32(key, keySource()) - - // Insert value into subbucket. - if err := b.Put(key, value); err != nil { - return err - } - } - - return nil - }); err != nil { - return err - } - } - return nil -} - -// Reads from the database. -func (cmd *BenchCommand) runReads(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - // Start profiling for reads. - if options.ProfileMode == "r" { - cmd.startProfiling(options) - } - - t := time.Now() - - var err error - switch options.ReadMode { - case "seq": - switch options.WriteMode { - case "seq-nest", "rnd-nest": - err = cmd.runReadsSequentialNested(db, options, results) - default: - err = cmd.runReadsSequential(db, options, results) - } - default: - return fmt.Errorf("invalid read mode: %s", options.ReadMode) - } - - // Save read time. - results.ReadDuration = time.Since(t) - - // Stop profiling for reads. - if options.ProfileMode == "rw" || options.ProfileMode == "r" { - cmd.stopProfiling() - } - - return err -} - -func (cmd *BenchCommand) runReadsSequential(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - return db.View(func(tx *bolt.Tx) error { - t := time.Now() - - for { - var count int - - c := tx.Bucket(benchBucketName).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - if v == nil { - return errors.New("invalid value") - } - count++ - } - - if options.WriteMode == "seq" && count != options.Iterations { - return fmt.Errorf("read seq: iter mismatch: expected %d, got %d", options.Iterations, count) - } - - results.ReadOps += count - - // Make sure we do this for at least a second. - if time.Since(t) >= time.Second { - break - } - } - - return nil - }) -} - -func (cmd *BenchCommand) runReadsSequentialNested(db *bolt.DB, options *BenchOptions, results *BenchResults) error { - return db.View(func(tx *bolt.Tx) error { - t := time.Now() - - for { - var count int - var top = tx.Bucket(benchBucketName) - if err := top.ForEach(func(name, _ []byte) error { - c := top.Bucket(name).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - if v == nil { - return ErrInvalidValue - } - count++ - } - return nil - }); err != nil { - return err - } - - if options.WriteMode == "seq-nest" && count != options.Iterations { - return fmt.Errorf("read seq-nest: iter mismatch: expected %d, got %d", options.Iterations, count) - } - - results.ReadOps += count - - // Make sure we do this for at least a second. - if time.Since(t) >= time.Second { - break - } - } - - return nil - }) -} - -// File handlers for the various profiles. -var cpuprofile, memprofile, blockprofile *os.File - -// Starts all profiles set on the options. -func (cmd *BenchCommand) startProfiling(options *BenchOptions) { - var err error - - // Start CPU profiling. - if options.CPUProfile != "" { - cpuprofile, err = os.Create(options.CPUProfile) - if err != nil { - fmt.Fprintf(cmd.Stderr, "bench: could not create cpu profile %q: %v\n", options.CPUProfile, err) - os.Exit(1) - } - pprof.StartCPUProfile(cpuprofile) - } - - // Start memory profiling. - if options.MemProfile != "" { - memprofile, err = os.Create(options.MemProfile) - if err != nil { - fmt.Fprintf(cmd.Stderr, "bench: could not create memory profile %q: %v\n", options.MemProfile, err) - os.Exit(1) - } - runtime.MemProfileRate = 4096 - } - - // Start fatal profiling. - if options.BlockProfile != "" { - blockprofile, err = os.Create(options.BlockProfile) - if err != nil { - fmt.Fprintf(cmd.Stderr, "bench: could not create block profile %q: %v\n", options.BlockProfile, err) - os.Exit(1) - } - runtime.SetBlockProfileRate(1) - } -} - -// Stops all profiles. -func (cmd *BenchCommand) stopProfiling() { - if cpuprofile != nil { - pprof.StopCPUProfile() - cpuprofile.Close() - cpuprofile = nil - } - - if memprofile != nil { - pprof.Lookup("heap").WriteTo(memprofile, 0) - memprofile.Close() - memprofile = nil - } - - if blockprofile != nil { - pprof.Lookup("block").WriteTo(blockprofile, 0) - blockprofile.Close() - blockprofile = nil - runtime.SetBlockProfileRate(0) - } -} - -// BenchOptions represents the set of options that can be passed to "bolt bench". -type BenchOptions struct { - ProfileMode string - WriteMode string - ReadMode string - Iterations int - BatchSize int - KeySize int - ValueSize int - CPUProfile string - MemProfile string - BlockProfile string - StatsInterval time.Duration - FillPercent float64 - NoSync bool - Work bool - Path string -} - -// BenchResults represents the performance results of the benchmark. -type BenchResults struct { - WriteOps int - WriteDuration time.Duration - ReadOps int - ReadDuration time.Duration -} - -// Returns the duration for a single write operation. -func (r *BenchResults) WriteOpDuration() time.Duration { - if r.WriteOps == 0 { - return 0 - } - return r.WriteDuration / time.Duration(r.WriteOps) -} - -// Returns average number of write operations that can be performed per second. -func (r *BenchResults) WriteOpsPerSecond() int { - var op = r.WriteOpDuration() - if op == 0 { - return 0 - } - return int(time.Second) / int(op) -} - -// Returns the duration for a single read operation. -func (r *BenchResults) ReadOpDuration() time.Duration { - if r.ReadOps == 0 { - return 0 - } - return r.ReadDuration / time.Duration(r.ReadOps) -} - -// Returns average number of read operations that can be performed per second. -func (r *BenchResults) ReadOpsPerSecond() int { - var op = r.ReadOpDuration() - if op == 0 { - return 0 - } - return int(time.Second) / int(op) -} - -type PageError struct { - ID int - Err error -} - -func (e *PageError) Error() string { - return fmt.Sprintf("page error: id=%d, err=%s", e.ID, e.Err) -} - -// isPrintable returns true if the string is valid unicode and contains only printable runes. -func isPrintable(s string) bool { - if !utf8.ValidString(s) { - return false - } - for _, ch := range s { - if !unicode.IsPrint(ch) { - return false - } - } - return true -} - -// ReadPage reads page info & full page data from a path. -// This is not transactionally safe. -func ReadPage(path string, pageID int) (*page, []byte, error) { - // Find page size. - pageSize, err := ReadPageSize(path) - if err != nil { - return nil, nil, fmt.Errorf("read page size: %s", err) - } - - // Open database file. - f, err := os.Open(path) - if err != nil { - return nil, nil, err - } - defer f.Close() - - // Read one block into buffer. - buf := make([]byte, pageSize) - if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { - return nil, nil, err - } else if n != len(buf) { - return nil, nil, io.ErrUnexpectedEOF - } - - // Determine total number of blocks. - p := (*page)(unsafe.Pointer(&buf[0])) - overflowN := p.overflow - - // Re-read entire page (with overflow) into buffer. - buf = make([]byte, (int(overflowN)+1)*pageSize) - if n, err := f.ReadAt(buf, int64(pageID*pageSize)); err != nil { - return nil, nil, err - } else if n != len(buf) { - return nil, nil, io.ErrUnexpectedEOF - } - p = (*page)(unsafe.Pointer(&buf[0])) - - return p, buf, nil -} - -// ReadPageSize reads page size a path. -// This is not transactionally safe. -func ReadPageSize(path string) (int, error) { - // Open database file. - f, err := os.Open(path) - if err != nil { - return 0, err - } - defer f.Close() - - // Read 4KB chunk. - buf := make([]byte, 4096) - if _, err := io.ReadFull(f, buf); err != nil { - return 0, err - } - - // Read page size from metadata. - m := (*meta)(unsafe.Pointer(&buf[PageHeaderSize])) - return int(m.pageSize), nil -} - -// atois parses a slice of strings into integers. -func atois(strs []string) ([]int, error) { - var a []int - for _, str := range strs { - i, err := strconv.Atoi(str) - if err != nil { - return nil, err - } - a = append(a, i) - } - return a, nil -} - -// DO NOT EDIT. Copied from the "bolt" package. -const maxAllocSize = 0xFFFFFFF - -// DO NOT EDIT. Copied from the "bolt" package. -const ( - branchPageFlag = 0x01 - leafPageFlag = 0x02 - metaPageFlag = 0x04 - freelistPageFlag = 0x10 -) - -// DO NOT EDIT. Copied from the "bolt" package. -const bucketLeafFlag = 0x01 - -// DO NOT EDIT. Copied from the "bolt" package. -type pgid uint64 - -// DO NOT EDIT. Copied from the "bolt" package. -type txid uint64 - -// DO NOT EDIT. Copied from the "bolt" package. -type meta struct { - magic uint32 - version uint32 - pageSize uint32 - flags uint32 - root bucket - freelist pgid - pgid pgid - txid txid - checksum uint64 -} - -// DO NOT EDIT. Copied from the "bolt" package. -type bucket struct { - root pgid - sequence uint64 -} - -// DO NOT EDIT. Copied from the "bolt" package. -type page struct { - id pgid - flags uint16 - count uint16 - overflow uint32 - ptr uintptr -} - -// DO NOT EDIT. Copied from the "bolt" package. -func (p *page) Type() string { - if (p.flags & branchPageFlag) != 0 { - return "branch" - } else if (p.flags & leafPageFlag) != 0 { - return "leaf" - } else if (p.flags & metaPageFlag) != 0 { - return "meta" - } else if (p.flags & freelistPageFlag) != 0 { - return "freelist" - } - return fmt.Sprintf("unknown<%02x>", p.flags) -} - -// DO NOT EDIT. Copied from the "bolt" package. -func (p *page) leafPageElement(index uint16) *leafPageElement { - n := &((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[index] - return n -} - -// DO NOT EDIT. Copied from the "bolt" package. -func (p *page) branchPageElement(index uint16) *branchPageElement { - return &((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[index] -} - -// DO NOT EDIT. Copied from the "bolt" package. -type branchPageElement struct { - pos uint32 - ksize uint32 - pgid pgid -} - -// DO NOT EDIT. Copied from the "bolt" package. -func (n *branchPageElement) key() []byte { - buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return buf[n.pos : n.pos+n.ksize] -} - -// DO NOT EDIT. Copied from the "bolt" package. -type leafPageElement struct { - flags uint32 - pos uint32 - ksize uint32 - vsize uint32 -} - -// DO NOT EDIT. Copied from the "bolt" package. -func (n *leafPageElement) key() []byte { - buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return buf[n.pos : n.pos+n.ksize] -} - -// DO NOT EDIT. Copied from the "bolt" package. -func (n *leafPageElement) value() []byte { - buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return buf[n.pos+n.ksize : n.pos+n.ksize+n.vsize] -} diff --git a/vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go b/vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go deleted file mode 100644 index c378b790..00000000 --- a/vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package main_test - -import ( - "bytes" - "io/ioutil" - "os" - "strconv" - "testing" - - "github.com/boltdb/bolt" - "github.com/boltdb/bolt/cmd/bolt" -) - -// Ensure the "info" command can print information about a database. -func TestInfoCommand_Run(t *testing.T) { - db := MustOpen(0666, nil) - db.DB.Close() - defer db.Close() - - // Run the info command. - m := NewMain() - if err := m.Run("info", db.Path); err != nil { - t.Fatal(err) - } -} - -// Ensure the "stats" command executes correctly with an empty database. -func TestStatsCommand_Run_EmptyDatabase(t *testing.T) { - // Ignore - if os.Getpagesize() != 4096 { - t.Skip("system does not use 4KB page size") - } - - db := MustOpen(0666, nil) - defer db.Close() - db.DB.Close() - - // Generate expected result. - exp := "Aggregate statistics for 0 buckets\n\n" + - "Page count statistics\n" + - "\tNumber of logical branch pages: 0\n" + - "\tNumber of physical branch overflow pages: 0\n" + - "\tNumber of logical leaf pages: 0\n" + - "\tNumber of physical leaf overflow pages: 0\n" + - "Tree statistics\n" + - "\tNumber of keys/value pairs: 0\n" + - "\tNumber of levels in B+tree: 0\n" + - "Page size utilization\n" + - "\tBytes allocated for physical branch pages: 0\n" + - "\tBytes actually used for branch data: 0 (0%)\n" + - "\tBytes allocated for physical leaf pages: 0\n" + - "\tBytes actually used for leaf data: 0 (0%)\n" + - "Bucket statistics\n" + - "\tTotal number of buckets: 0\n" + - "\tTotal number on inlined buckets: 0 (0%)\n" + - "\tBytes used for inlined buckets: 0 (0%)\n" - - // Run the command. - m := NewMain() - if err := m.Run("stats", db.Path); err != nil { - t.Fatal(err) - } else if m.Stdout.String() != exp { - t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) - } -} - -// Ensure the "stats" command can execute correctly. -func TestStatsCommand_Run(t *testing.T) { - // Ignore - if os.Getpagesize() != 4096 { - t.Skip("system does not use 4KB page size") - } - - db := MustOpen(0666, nil) - defer db.Close() - - if err := db.Update(func(tx *bolt.Tx) error { - // Create "foo" bucket. - b, err := tx.CreateBucket([]byte("foo")) - if err != nil { - return err - } - for i := 0; i < 10; i++ { - if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { - return err - } - } - - // Create "bar" bucket. - b, err = tx.CreateBucket([]byte("bar")) - if err != nil { - return err - } - for i := 0; i < 100; i++ { - if err := b.Put([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))); err != nil { - return err - } - } - - // Create "baz" bucket. - b, err = tx.CreateBucket([]byte("baz")) - if err != nil { - return err - } - if err := b.Put([]byte("key"), []byte("value")); err != nil { - return err - } - - return nil - }); err != nil { - t.Fatal(err) - } - db.DB.Close() - - // Generate expected result. - exp := "Aggregate statistics for 3 buckets\n\n" + - "Page count statistics\n" + - "\tNumber of logical branch pages: 0\n" + - "\tNumber of physical branch overflow pages: 0\n" + - "\tNumber of logical leaf pages: 1\n" + - "\tNumber of physical leaf overflow pages: 0\n" + - "Tree statistics\n" + - "\tNumber of keys/value pairs: 111\n" + - "\tNumber of levels in B+tree: 1\n" + - "Page size utilization\n" + - "\tBytes allocated for physical branch pages: 0\n" + - "\tBytes actually used for branch data: 0 (0%)\n" + - "\tBytes allocated for physical leaf pages: 4096\n" + - "\tBytes actually used for leaf data: 1996 (48%)\n" + - "Bucket statistics\n" + - "\tTotal number of buckets: 3\n" + - "\tTotal number on inlined buckets: 2 (66%)\n" + - "\tBytes used for inlined buckets: 236 (11%)\n" - - // Run the command. - m := NewMain() - if err := m.Run("stats", db.Path); err != nil { - t.Fatal(err) - } else if m.Stdout.String() != exp { - t.Fatalf("unexpected stdout:\n\n%s", m.Stdout.String()) - } -} - -// Main represents a test wrapper for main.Main that records output. -type Main struct { - *main.Main - Stdin bytes.Buffer - Stdout bytes.Buffer - Stderr bytes.Buffer -} - -// NewMain returns a new instance of Main. -func NewMain() *Main { - m := &Main{Main: main.NewMain()} - m.Main.Stdin = &m.Stdin - m.Main.Stdout = &m.Stdout - m.Main.Stderr = &m.Stderr - return m -} - -// MustOpen creates a Bolt database in a temporary location. -func MustOpen(mode os.FileMode, options *bolt.Options) *DB { - // Create temporary path. - f, _ := ioutil.TempFile("", "bolt-") - f.Close() - os.Remove(f.Name()) - - db, err := bolt.Open(f.Name(), mode, options) - if err != nil { - panic(err.Error()) - } - return &DB{DB: db, Path: f.Name()} -} - -// DB is a test wrapper for bolt.DB. -type DB struct { - *bolt.DB - Path string -} - -// Close closes and removes the database. -func (db *DB) Close() error { - defer os.Remove(db.Path) - return db.DB.Close() -} diff --git a/vendor/github.com/boltdb/bolt/cursor_test.go b/vendor/github.com/boltdb/bolt/cursor_test.go deleted file mode 100644 index 562d60f9..00000000 --- a/vendor/github.com/boltdb/bolt/cursor_test.go +++ /dev/null @@ -1,817 +0,0 @@ -package bolt_test - -import ( - "bytes" - "encoding/binary" - "fmt" - "log" - "os" - "reflect" - "sort" - "testing" - "testing/quick" - - "github.com/boltdb/bolt" -) - -// Ensure that a cursor can return a reference to the bucket that created it. -func TestCursor_Bucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if cb := b.Cursor().Bucket(); !reflect.DeepEqual(cb, b) { - t.Fatal("cursor bucket mismatch") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx cursor can seek to the appropriate keys. -func TestCursor_Seek(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("0001")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("bar"), []byte("0002")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte("0003")); err != nil { - t.Fatal(err) - } - - if _, err := b.CreateBucket([]byte("bkt")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte("widgets")).Cursor() - - // Exact match should go to the key. - if k, v := c.Seek([]byte("bar")); !bytes.Equal(k, []byte("bar")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte("0002")) { - t.Fatalf("unexpected value: %v", v) - } - - // Inexact match should go to the next key. - if k, v := c.Seek([]byte("bas")); !bytes.Equal(k, []byte("baz")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte("0003")) { - t.Fatalf("unexpected value: %v", v) - } - - // Low key should go to the first key. - if k, v := c.Seek([]byte("")); !bytes.Equal(k, []byte("bar")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte("0002")) { - t.Fatalf("unexpected value: %v", v) - } - - // High key should return no key. - if k, v := c.Seek([]byte("zzz")); k != nil { - t.Fatalf("expected nil key: %v", k) - } else if v != nil { - t.Fatalf("expected nil value: %v", v) - } - - // Buckets should return their key but no value. - if k, v := c.Seek([]byte("bkt")); !bytes.Equal(k, []byte("bkt")) { - t.Fatalf("unexpected key: %v", k) - } else if v != nil { - t.Fatalf("expected nil value: %v", v) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -func TestCursor_Delete(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - const count = 1000 - - // Insert every other key between 0 and $count. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - for i := 0; i < count; i += 1 { - k := make([]byte, 8) - binary.BigEndian.PutUint64(k, uint64(i)) - if err := b.Put(k, make([]byte, 100)); err != nil { - t.Fatal(err) - } - } - if _, err := b.CreateBucket([]byte("sub")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte("widgets")).Cursor() - bound := make([]byte, 8) - binary.BigEndian.PutUint64(bound, uint64(count/2)) - for key, _ := c.First(); bytes.Compare(key, bound) < 0; key, _ = c.Next() { - if err := c.Delete(); err != nil { - t.Fatal(err) - } - } - - c.Seek([]byte("sub")) - if err := c.Delete(); err != bolt.ErrIncompatibleValue { - t.Fatalf("unexpected error: %s", err) - } - - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - stats := tx.Bucket([]byte("widgets")).Stats() - if stats.KeyN != count/2+1 { - t.Fatalf("unexpected KeyN: %d", stats.KeyN) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx cursor can seek to the appropriate keys when there are a -// large number of keys. This test also checks that seek will always move -// forward to the next key. -// -// Related: https://github.com/boltdb/bolt/pull/187 -func TestCursor_Seek_Large(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var count = 10000 - - // Insert every other key between 0 and $count. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < count; i += 100 { - for j := i; j < i+100; j += 2 { - k := make([]byte, 8) - binary.BigEndian.PutUint64(k, uint64(j)) - if err := b.Put(k, make([]byte, 100)); err != nil { - t.Fatal(err) - } - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte("widgets")).Cursor() - for i := 0; i < count; i++ { - seek := make([]byte, 8) - binary.BigEndian.PutUint64(seek, uint64(i)) - - k, _ := c.Seek(seek) - - // The last seek is beyond the end of the the range so - // it should return nil. - if i == count-1 { - if k != nil { - t.Fatal("expected nil key") - } - continue - } - - // Otherwise we should seek to the exact key or the next key. - num := binary.BigEndian.Uint64(k) - if i%2 == 0 { - if num != uint64(i) { - t.Fatalf("unexpected num: %d", num) - } - } else { - if num != uint64(i+1) { - t.Fatalf("unexpected num: %d", num) - } - } - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a cursor can iterate over an empty bucket without error. -func TestCursor_EmptyBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte("widgets")).Cursor() - k, v := c.First() - if k != nil { - t.Fatalf("unexpected key: %v", k) - } else if v != nil { - t.Fatalf("unexpected value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx cursor can reverse iterate over an empty bucket without error. -func TestCursor_EmptyBucketReverse(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - t.Fatal(err) - } - if err := db.View(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte("widgets")).Cursor() - k, v := c.Last() - if k != nil { - t.Fatalf("unexpected key: %v", k) - } else if v != nil { - t.Fatalf("unexpected value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx cursor can iterate over a single root with a couple elements. -func TestCursor_Iterate_Leaf(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte{}); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte{0}); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("bar"), []byte{1}); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - tx, err := db.Begin(false) - if err != nil { - t.Fatal(err) - } - defer func() { _ = tx.Rollback() }() - - c := tx.Bucket([]byte("widgets")).Cursor() - - k, v := c.First() - if !bytes.Equal(k, []byte("bar")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte{1}) { - t.Fatalf("unexpected value: %v", v) - } - - k, v = c.Next() - if !bytes.Equal(k, []byte("baz")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte{}) { - t.Fatalf("unexpected value: %v", v) - } - - k, v = c.Next() - if !bytes.Equal(k, []byte("foo")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte{0}) { - t.Fatalf("unexpected value: %v", v) - } - - k, v = c.Next() - if k != nil { - t.Fatalf("expected nil key: %v", k) - } else if v != nil { - t.Fatalf("expected nil value: %v", v) - } - - k, v = c.Next() - if k != nil { - t.Fatalf("expected nil key: %v", k) - } else if v != nil { - t.Fatalf("expected nil value: %v", v) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx cursor can iterate in reverse over a single root with a couple elements. -func TestCursor_LeafRootReverse(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte{}); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte{0}); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("bar"), []byte{1}); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - tx, err := db.Begin(false) - if err != nil { - t.Fatal(err) - } - c := tx.Bucket([]byte("widgets")).Cursor() - - if k, v := c.Last(); !bytes.Equal(k, []byte("foo")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte{0}) { - t.Fatalf("unexpected value: %v", v) - } - - if k, v := c.Prev(); !bytes.Equal(k, []byte("baz")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte{}) { - t.Fatalf("unexpected value: %v", v) - } - - if k, v := c.Prev(); !bytes.Equal(k, []byte("bar")) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, []byte{1}) { - t.Fatalf("unexpected value: %v", v) - } - - if k, v := c.Prev(); k != nil { - t.Fatalf("expected nil key: %v", k) - } else if v != nil { - t.Fatalf("expected nil value: %v", v) - } - - if k, v := c.Prev(); k != nil { - t.Fatalf("expected nil key: %v", k) - } else if v != nil { - t.Fatalf("expected nil value: %v", v) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx cursor can restart from the beginning. -func TestCursor_Restart(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("bar"), []byte{}); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte{}); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - tx, err := db.Begin(false) - if err != nil { - t.Fatal(err) - } - c := tx.Bucket([]byte("widgets")).Cursor() - - if k, _ := c.First(); !bytes.Equal(k, []byte("bar")) { - t.Fatalf("unexpected key: %v", k) - } - if k, _ := c.Next(); !bytes.Equal(k, []byte("foo")) { - t.Fatalf("unexpected key: %v", k) - } - - if k, _ := c.First(); !bytes.Equal(k, []byte("bar")) { - t.Fatalf("unexpected key: %v", k) - } - if k, _ := c.Next(); !bytes.Equal(k, []byte("foo")) { - t.Fatalf("unexpected key: %v", k) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } -} - -// Ensure that a cursor can skip over empty pages that have been deleted. -func TestCursor_First_EmptyPages(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - // Create 1000 keys in the "widgets" bucket. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - for i := 0; i < 1000; i++ { - if err := b.Put(u64tob(uint64(i)), []byte{}); err != nil { - t.Fatal(err) - } - } - - return nil - }); err != nil { - t.Fatal(err) - } - - // Delete half the keys and then try to iterate. - if err := db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for i := 0; i < 600; i++ { - if err := b.Delete(u64tob(uint64(i))); err != nil { - t.Fatal(err) - } - } - - c := b.Cursor() - var n int - for k, _ := c.First(); k != nil; k, _ = c.Next() { - n++ - } - if n != 400 { - t.Fatalf("unexpected key count: %d", n) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx can iterate over all elements in a bucket. -func TestCursor_QuickCheck(t *testing.T) { - f := func(items testdata) bool { - db := MustOpenDB() - defer db.MustClose() - - // Bulk insert all values. - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - for _, item := range items { - if err := b.Put(item.Key, item.Value); err != nil { - t.Fatal(err) - } - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - - // Sort test data. - sort.Sort(items) - - // Iterate over all items and check consistency. - var index = 0 - tx, err = db.Begin(false) - if err != nil { - t.Fatal(err) - } - - c := tx.Bucket([]byte("widgets")).Cursor() - for k, v := c.First(); k != nil && index < len(items); k, v = c.Next() { - if !bytes.Equal(k, items[index].Key) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, items[index].Value) { - t.Fatalf("unexpected value: %v", v) - } - index++ - } - if len(items) != index { - t.Fatalf("unexpected item count: %v, expected %v", len(items), index) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - - return true - } - if err := quick.Check(f, qconfig()); err != nil { - t.Error(err) - } -} - -// Ensure that a transaction can iterate over all elements in a bucket in reverse. -func TestCursor_QuickCheck_Reverse(t *testing.T) { - f := func(items testdata) bool { - db := MustOpenDB() - defer db.MustClose() - - // Bulk insert all values. - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - for _, item := range items { - if err := b.Put(item.Key, item.Value); err != nil { - t.Fatal(err) - } - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - - // Sort test data. - sort.Sort(revtestdata(items)) - - // Iterate over all items and check consistency. - var index = 0 - tx, err = db.Begin(false) - if err != nil { - t.Fatal(err) - } - c := tx.Bucket([]byte("widgets")).Cursor() - for k, v := c.Last(); k != nil && index < len(items); k, v = c.Prev() { - if !bytes.Equal(k, items[index].Key) { - t.Fatalf("unexpected key: %v", k) - } else if !bytes.Equal(v, items[index].Value) { - t.Fatalf("unexpected value: %v", v) - } - index++ - } - if len(items) != index { - t.Fatalf("unexpected item count: %v, expected %v", len(items), index) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - - return true - } - if err := quick.Check(f, qconfig()); err != nil { - t.Error(err) - } -} - -// Ensure that a Tx cursor can iterate over subbuckets. -func TestCursor_QuickCheck_BucketsOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if _, err := b.CreateBucket([]byte("foo")); err != nil { - t.Fatal(err) - } - if _, err := b.CreateBucket([]byte("bar")); err != nil { - t.Fatal(err) - } - if _, err := b.CreateBucket([]byte("baz")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - var names []string - c := tx.Bucket([]byte("widgets")).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - names = append(names, string(k)) - if v != nil { - t.Fatalf("unexpected value: %v", v) - } - } - if !reflect.DeepEqual(names, []string{"bar", "baz", "foo"}) { - t.Fatalf("unexpected names: %+v", names) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx cursor can reverse iterate over subbuckets. -func TestCursor_QuickCheck_BucketsOnly_Reverse(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if _, err := b.CreateBucket([]byte("foo")); err != nil { - t.Fatal(err) - } - if _, err := b.CreateBucket([]byte("bar")); err != nil { - t.Fatal(err) - } - if _, err := b.CreateBucket([]byte("baz")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - var names []string - c := tx.Bucket([]byte("widgets")).Cursor() - for k, v := c.Last(); k != nil; k, v = c.Prev() { - names = append(names, string(k)) - if v != nil { - t.Fatalf("unexpected value: %v", v) - } - } - if !reflect.DeepEqual(names, []string{"foo", "baz", "bar"}) { - t.Fatalf("unexpected names: %+v", names) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -func ExampleCursor() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Start a read-write transaction. - if err := db.Update(func(tx *bolt.Tx) error { - // Create a new bucket. - b, err := tx.CreateBucket([]byte("animals")) - if err != nil { - return err - } - - // Insert data into a bucket. - if err := b.Put([]byte("dog"), []byte("fun")); err != nil { - log.Fatal(err) - } - if err := b.Put([]byte("cat"), []byte("lame")); err != nil { - log.Fatal(err) - } - if err := b.Put([]byte("liger"), []byte("awesome")); err != nil { - log.Fatal(err) - } - - // Create a cursor for iteration. - c := b.Cursor() - - // Iterate over items in sorted key order. This starts from the - // first key/value pair and updates the k/v variables to the - // next key/value on each iteration. - // - // The loop finishes at the end of the cursor when a nil key is returned. - for k, v := c.First(); k != nil; k, v = c.Next() { - fmt.Printf("A %s is %s.\n", k, v) - } - - return nil - }); err != nil { - log.Fatal(err) - } - - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // A cat is lame. - // A dog is fun. - // A liger is awesome. -} - -func ExampleCursor_reverse() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Start a read-write transaction. - if err := db.Update(func(tx *bolt.Tx) error { - // Create a new bucket. - b, err := tx.CreateBucket([]byte("animals")) - if err != nil { - return err - } - - // Insert data into a bucket. - if err := b.Put([]byte("dog"), []byte("fun")); err != nil { - log.Fatal(err) - } - if err := b.Put([]byte("cat"), []byte("lame")); err != nil { - log.Fatal(err) - } - if err := b.Put([]byte("liger"), []byte("awesome")); err != nil { - log.Fatal(err) - } - - // Create a cursor for iteration. - c := b.Cursor() - - // Iterate over items in reverse sorted key order. This starts - // from the last key/value pair and updates the k/v variables to - // the previous key/value on each iteration. - // - // The loop finishes at the beginning of the cursor when a nil key - // is returned. - for k, v := c.Last(); k != nil; k, v = c.Prev() { - fmt.Printf("A %s is %s.\n", k, v) - } - - return nil - }); err != nil { - log.Fatal(err) - } - - // Close the database to release the file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // A liger is awesome. - // A dog is fun. - // A cat is lame. -} diff --git a/vendor/github.com/boltdb/bolt/db.go b/vendor/github.com/boltdb/bolt/db.go index 0f1e1bc3..f352ff14 100644 --- a/vendor/github.com/boltdb/bolt/db.go +++ b/vendor/github.com/boltdb/bolt/db.go @@ -36,6 +36,9 @@ const ( DefaultAllocSize = 16 * 1024 * 1024 ) +// default page size for db is set to the OS page size. +var defaultPageSize = os.Getpagesize() + // DB represents a collection of buckets persisted to a file on disk. // All data access is performed through transactions which can be obtained through the DB. // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. @@ -93,7 +96,8 @@ type DB struct { path string file *os.File - dataref []byte // mmap'ed readonly, write throws SEGV + lockfile *os.File // windows only + dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int filesz int // current on disk file size @@ -106,6 +110,8 @@ type DB struct { freelist *freelist stats Stats + pagePool sync.Pool + batchMu sync.Mutex batch *batch @@ -177,7 +183,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // if !options.ReadOnly. // The database file is locked using the shared lock (more than one process may // hold a lock at the same time) otherwise (options.ReadOnly is set). - if err := flock(db.file, !db.readOnly, options.Timeout); err != nil { + if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil { _ = db.close() return nil, err } @@ -199,12 +205,27 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { if _, err := db.file.ReadAt(buf[:], 0); err == nil { m := db.pageInBuffer(buf[:], 0).meta() if err := m.validate(); err != nil { - return nil, err + // If we can't read the page size, we can assume it's the same + // as the OS -- since that's how the page size was chosen in the + // first place. + // + // If the first page is invalid and this OS uses a different + // page size than what the database was created with then we + // are out of luck and cannot access the database. + db.pageSize = os.Getpagesize() + } else { + db.pageSize = int(m.pageSize) } - db.pageSize = int(m.pageSize) } } + // Initialize page pool. + db.pagePool = sync.Pool{ + New: func() interface{} { + return make([]byte, db.pageSize) + }, + } + // Memory map the data file. if err := db.mmap(options.InitialMmapSize); err != nil { _ = db.close() @@ -261,12 +282,13 @@ func (db *DB) mmap(minsz int) error { db.meta0 = db.page(0).meta() db.meta1 = db.page(1).meta() - // Validate the meta pages. - if err := db.meta0.validate(); err != nil { - return err - } - if err := db.meta1.validate(); err != nil { - return err + // Validate the meta pages. We only return an error if both meta pages fail + // validation, since meta0 failing validation means that it wasn't saved + // properly -- but we can recover using meta1. And vice-versa. + err0 := db.meta0.validate() + err1 := db.meta1.validate() + if err0 != nil && err1 != nil { + return err0 } return nil @@ -338,6 +360,7 @@ func (db *DB) init() error { m.root = bucket{root: 3} m.pgid = 4 m.txid = txid(i) + m.checksum = m.sum64() } // Write an empty freelist at page 3. @@ -379,10 +402,13 @@ func (db *DB) Close() error { } func (db *DB) close() error { + if !db.opened { + return nil + } + db.opened = false db.freelist = nil - db.path = "" // Clear ops. db.ops.writeAt = nil @@ -397,7 +423,7 @@ func (db *DB) close() error { // No need to unlock read-only file. if !db.readOnly { // Unlock the file. - if err := funlock(db.file); err != nil { + if err := funlock(db); err != nil { log.Printf("bolt.Close(): funlock error: %s", err) } } @@ -409,6 +435,7 @@ func (db *DB) close() error { db.file = nil } + db.path = "" return nil } @@ -525,7 +552,10 @@ func (db *DB) removeTx(tx *Tx) { // Remove the transaction. for i, t := range db.txs { if t == tx { - db.txs = append(db.txs[:i], db.txs[i+1:]...) + last := len(db.txs) - 1 + db.txs[i] = db.txs[last] + db.txs[last] = nil + db.txs = db.txs[:last] break } } @@ -773,16 +803,37 @@ func (db *DB) pageInBuffer(b []byte, id pgid) *page { // meta retrieves the current meta page reference. func (db *DB) meta() *meta { - if db.meta0.txid > db.meta1.txid { - return db.meta0 + // We have to return the meta with the highest txid which doesn't fail + // validation. Otherwise, we can cause errors when in fact the database is + // in a consistent state. metaA is the one with the higher txid. + metaA := db.meta0 + metaB := db.meta1 + if db.meta1.txid > db.meta0.txid { + metaA = db.meta1 + metaB = db.meta0 } - return db.meta1 + + // Use higher meta page if valid. Otherwise fallback to previous, if valid. + if err := metaA.validate(); err == nil { + return metaA + } else if err := metaB.validate(); err == nil { + return metaB + } + + // This should never be reached, because both meta1 and meta0 were validated + // on mmap() and we do fsync() on every write. + panic("bolt.DB.meta(): invalid meta pages") } // allocate returns a contiguous block of memory starting at a given page. func (db *DB) allocate(count int) (*page, error) { // Allocate a temporary buffer for the page. - buf := make([]byte, count*db.pageSize) + var buf []byte + if count == 1 { + buf = db.pagePool.Get().([]byte) + } else { + buf = make([]byte, count*db.pageSize) + } p := (*page)(unsafe.Pointer(&buf[0])) p.overflow = uint32(count - 1) @@ -824,8 +875,10 @@ func (db *DB) grow(sz int) error { // Truncate and fsync to ensure file size metadata is flushed. // https://github.com/boltdb/bolt/issues/284 if !db.NoGrowSync && !db.readOnly { - if err := db.file.Truncate(int64(sz)); err != nil { - return fmt.Errorf("file resize error: %s", err) + if runtime.GOOS != "windows" { + if err := db.file.Truncate(int64(sz)); err != nil { + return fmt.Errorf("file resize error: %s", err) + } } if err := db.file.Sync(); err != nil { return fmt.Errorf("file sync error: %s", err) @@ -902,7 +955,7 @@ func (s *Stats) Sub(other *Stats) Stats { diff.PendingPageN = s.PendingPageN diff.FreeAlloc = s.FreeAlloc diff.FreelistInuse = s.FreelistInuse - diff.TxN = other.TxN - s.TxN + diff.TxN = s.TxN - other.TxN diff.TxStats = s.TxStats.Sub(&other.TxStats) return diff } @@ -930,12 +983,12 @@ type meta struct { // validate checks the marker bytes and version of the meta page to ensure it matches this binary. func (m *meta) validate() error { - if m.checksum != 0 && m.checksum != m.sum64() { - return ErrChecksum - } else if m.magic != magic { + if m.magic != magic { return ErrInvalid } else if m.version != version { return ErrVersionMismatch + } else if m.checksum != 0 && m.checksum != m.sum64() { + return ErrChecksum } return nil } diff --git a/vendor/github.com/boltdb/bolt/db_test.go b/vendor/github.com/boltdb/bolt/db_test.go deleted file mode 100644 index e31b8658..00000000 --- a/vendor/github.com/boltdb/bolt/db_test.go +++ /dev/null @@ -1,1684 +0,0 @@ -package bolt_test - -import ( - "bytes" - "encoding/binary" - "errors" - "flag" - "fmt" - "hash/fnv" - "io/ioutil" - "log" - "os" - "path/filepath" - "regexp" - "runtime" - "sort" - "strings" - "sync" - "testing" - "time" - "unsafe" - - "github.com/boltdb/bolt" -) - -var statsFlag = flag.Bool("stats", false, "show performance stats") - -// version is the data file format version. -const version = 2 - -// magic is the marker value to indicate that a file is a Bolt DB. -const magic uint32 = 0xED0CDAED - -// pageSize is the size of one page in the data file. -const pageSize = 4096 - -// pageHeaderSize is the size of a page header. -const pageHeaderSize = 16 - -// meta represents a simplified version of a database meta page for testing. -type meta struct { - magic uint32 - version uint32 - _ uint32 - _ uint32 - _ [16]byte - _ uint64 - _ uint64 - _ uint64 - checksum uint64 -} - -// Ensure that a database can be opened without error. -func TestOpen(t *testing.T) { - path := tempfile() - db, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } else if db == nil { - t.Fatal("expected db") - } - - if s := db.Path(); s != path { - t.Fatalf("unexpected path: %s", s) - } - - if err := db.Close(); err != nil { - t.Fatal(err) - } -} - -// Ensure that opening a database with a blank path returns an error. -func TestOpen_ErrPathRequired(t *testing.T) { - _, err := bolt.Open("", 0666, nil) - if err == nil { - t.Fatalf("expected error") - } -} - -// Ensure that opening a database with a bad path returns an error. -func TestOpen_ErrNotExists(t *testing.T) { - _, err := bolt.Open(filepath.Join(tempfile(), "bad-path"), 0666, nil) - if err == nil { - t.Fatal("expected error") - } -} - -// Ensure that opening a file with wrong checksum returns ErrChecksum. -func TestOpen_ErrChecksum(t *testing.T) { - buf := make([]byte, pageSize) - meta := (*meta)(unsafe.Pointer(&buf[0])) - meta.magic = magic - meta.version = version - meta.checksum = 123 - - path := tempfile() - f, err := os.Create(path) - if err != nil { - t.Fatal(err) - } - if _, err := f.WriteAt(buf, pageHeaderSize); err != nil { - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - defer os.Remove(path) - - if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrChecksum { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that opening a file that is not a Bolt database returns ErrInvalid. -func TestOpen_ErrInvalid(t *testing.T) { - path := tempfile() - - f, err := os.Create(path) - if err != nil { - t.Fatal(err) - } - if _, err := fmt.Fprintln(f, "this is not a bolt database"); err != nil { - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - defer os.Remove(path) - - if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrInvalid { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that opening a file created with a different version of Bolt returns -// ErrVersionMismatch. -func TestOpen_ErrVersionMismatch(t *testing.T) { - buf := make([]byte, pageSize) - meta := (*meta)(unsafe.Pointer(&buf[0])) - meta.magic = magic - meta.version = version + 100 - - path := tempfile() - f, err := os.Create(path) - if err != nil { - t.Fatal(err) - } - if _, err := f.WriteAt(buf, pageHeaderSize); err != nil { - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } - defer os.Remove(path) - - if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrVersionMismatch { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that opening an already open database file will timeout. -func TestOpen_Timeout(t *testing.T) { - if runtime.GOOS == "solaris" { - t.Skip("solaris fcntl locks don't support intra-process locking") - } - - path := tempfile() - - // Open a data file. - db0, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } else if db0 == nil { - t.Fatal("expected database") - } - - // Attempt to open the database again. - start := time.Now() - db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 100 * time.Millisecond}) - if err != bolt.ErrTimeout { - t.Fatalf("unexpected timeout: %s", err) - } else if db1 != nil { - t.Fatal("unexpected database") - } else if time.Since(start) <= 100*time.Millisecond { - t.Fatal("expected to wait at least timeout duration") - } - - if err := db0.Close(); err != nil { - t.Fatal(err) - } -} - -// Ensure that opening an already open database file will wait until its closed. -func TestOpen_Wait(t *testing.T) { - if runtime.GOOS == "solaris" { - t.Skip("solaris fcntl locks don't support intra-process locking") - } - - path := tempfile() - - // Open a data file. - db0, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - - // Close it in just a bit. - time.AfterFunc(100*time.Millisecond, func() { _ = db0.Close() }) - - // Attempt to open the database again. - start := time.Now() - db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 200 * time.Millisecond}) - if err != nil { - t.Fatal(err) - } else if time.Since(start) <= 100*time.Millisecond { - t.Fatal("expected to wait at least timeout duration") - } - - if err := db1.Close(); err != nil { - t.Fatal(err) - } -} - -// Ensure that opening a database does not increase its size. -// https://github.com/boltdb/bolt/issues/291 -func TestOpen_Size(t *testing.T) { - // Open a data file. - db := MustOpenDB() - path := db.Path() - defer db.MustClose() - - pagesize := db.Info().PageSize - - // Insert until we get above the minimum 4MB size. - if err := db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists([]byte("data")) - for i := 0; i < 10000; i++ { - if err := b.Put([]byte(fmt.Sprintf("%04d", i)), make([]byte, 1000)); err != nil { - t.Fatal(err) - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Close database and grab the size. - if err := db.DB.Close(); err != nil { - t.Fatal(err) - } - sz := fileSize(path) - if sz == 0 { - t.Fatalf("unexpected new file size: %d", sz) - } - - // Reopen database, update, and check size again. - db0, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - if err := db0.Update(func(tx *bolt.Tx) error { - if err := tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - if err := db0.Close(); err != nil { - t.Fatal(err) - } - newSz := fileSize(path) - if newSz == 0 { - t.Fatalf("unexpected new file size: %d", newSz) - } - - // Compare the original size with the new size. - // db size might increase by a few page sizes due to the new small update. - if sz < newSz-5*int64(pagesize) { - t.Fatalf("unexpected file growth: %d => %d", sz, newSz) - } -} - -// Ensure that opening a database beyond the max step size does not increase its size. -// https://github.com/boltdb/bolt/issues/303 -func TestOpen_Size_Large(t *testing.T) { - if testing.Short() { - t.Skip("short mode") - } - - // Open a data file. - db := MustOpenDB() - path := db.Path() - defer db.MustClose() - - pagesize := db.Info().PageSize - - // Insert until we get above the minimum 4MB size. - var index uint64 - for i := 0; i < 10000; i++ { - if err := db.Update(func(tx *bolt.Tx) error { - b, _ := tx.CreateBucketIfNotExists([]byte("data")) - for j := 0; j < 1000; j++ { - if err := b.Put(u64tob(index), make([]byte, 50)); err != nil { - t.Fatal(err) - } - index++ - } - return nil - }); err != nil { - t.Fatal(err) - } - } - - // Close database and grab the size. - if err := db.DB.Close(); err != nil { - t.Fatal(err) - } - sz := fileSize(path) - if sz == 0 { - t.Fatalf("unexpected new file size: %d", sz) - } else if sz < (1 << 30) { - t.Fatalf("expected larger initial size: %d", sz) - } - - // Reopen database, update, and check size again. - db0, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - if err := db0.Update(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("data")).Put([]byte{0}, []byte{0}) - }); err != nil { - t.Fatal(err) - } - if err := db0.Close(); err != nil { - t.Fatal(err) - } - - newSz := fileSize(path) - if newSz == 0 { - t.Fatalf("unexpected new file size: %d", newSz) - } - - // Compare the original size with the new size. - // db size might increase by a few page sizes due to the new small update. - if sz < newSz-5*int64(pagesize) { - t.Fatalf("unexpected file growth: %d => %d", sz, newSz) - } -} - -// Ensure that a re-opened database is consistent. -func TestOpen_Check(t *testing.T) { - path := tempfile() - - db, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { - t.Fatal(err) - } - if err := db.Close(); err != nil { - t.Fatal(err) - } - - db, err = bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - if err := db.View(func(tx *bolt.Tx) error { return <-tx.Check() }); err != nil { - t.Fatal(err) - } - if err := db.Close(); err != nil { - t.Fatal(err) - } -} - -// Ensure that write errors to the meta file handler during initialization are returned. -func TestOpen_MetaInitWriteError(t *testing.T) { - t.Skip("pending") -} - -// Ensure that a database that is too small returns an error. -func TestOpen_FileTooSmall(t *testing.T) { - path := tempfile() - - db, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - if err := db.Close(); err != nil { - t.Fatal(err) - } - - // corrupt the database - if err := os.Truncate(path, int64(os.Getpagesize())); err != nil { - t.Fatal(err) - } - - db, err = bolt.Open(path, 0666, nil) - if err == nil || err.Error() != "file size too small" { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that a database can be opened in read-only mode by multiple processes -// and that a database can not be opened in read-write mode and in read-only -// mode at the same time. -func TestOpen_ReadOnly(t *testing.T) { - if runtime.GOOS == "solaris" { - t.Skip("solaris fcntl locks don't support intra-process locking") - } - - bucket, key, value := []byte(`bucket`), []byte(`key`), []byte(`value`) - - path := tempfile() - - // Open in read-write mode. - db, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } else if db.IsReadOnly() { - t.Fatal("db should not be in read only mode") - } - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket(bucket) - if err != nil { - return err - } - if err := b.Put(key, value); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - if err := db.Close(); err != nil { - t.Fatal(err) - } - - // Open in read-only mode. - db0, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) - if err != nil { - t.Fatal(err) - } - - // Opening in read-write mode should return an error. - if _, err = bolt.Open(path, 0666, &bolt.Options{Timeout: time.Millisecond * 100}); err == nil { - t.Fatal("expected error") - } - - // And again (in read-only mode). - db1, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) - if err != nil { - t.Fatal(err) - } - - // Verify both read-only databases are accessible. - for _, db := range []*bolt.DB{db0, db1} { - // Verify is is in read only mode indeed. - if !db.IsReadOnly() { - t.Fatal("expected read only mode") - } - - // Read-only databases should not allow updates. - if err := db.Update(func(*bolt.Tx) error { - panic(`should never get here`) - }); err != bolt.ErrDatabaseReadOnly { - t.Fatalf("unexpected error: %s", err) - } - - // Read-only databases should not allow beginning writable txns. - if _, err := db.Begin(true); err != bolt.ErrDatabaseReadOnly { - t.Fatalf("unexpected error: %s", err) - } - - // Verify the data. - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(bucket) - if b == nil { - return fmt.Errorf("expected bucket `%s`", string(bucket)) - } - - got := string(b.Get(key)) - expected := string(value) - if got != expected { - return fmt.Errorf("expected `%s`, got `%s`", expected, got) - } - return nil - }); err != nil { - t.Fatal(err) - } - } - - if err := db0.Close(); err != nil { - t.Fatal(err) - } - if err := db1.Close(); err != nil { - t.Fatal(err) - } -} - -// TestDB_Open_InitialMmapSize tests if having InitialMmapSize large enough -// to hold data from concurrent write transaction resolves the issue that -// read transaction blocks the write transaction and causes deadlock. -// This is a very hacky test since the mmap size is not exposed. -func TestDB_Open_InitialMmapSize(t *testing.T) { - path := tempfile() - defer os.Remove(path) - - initMmapSize := 1 << 31 // 2GB - testWriteSize := 1 << 27 // 134MB - - db, err := bolt.Open(path, 0666, &bolt.Options{InitialMmapSize: initMmapSize}) - if err != nil { - t.Fatal(err) - } - - // create a long-running read transaction - // that never gets closed while writing - rtx, err := db.Begin(false) - if err != nil { - t.Fatal(err) - } - - // create a write transaction - wtx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - - b, err := wtx.CreateBucket([]byte("test")) - if err != nil { - t.Fatal(err) - } - - // and commit a large write - err = b.Put([]byte("foo"), make([]byte, testWriteSize)) - if err != nil { - t.Fatal(err) - } - - done := make(chan struct{}) - - go func() { - if err := wtx.Commit(); err != nil { - t.Fatal(err) - } - done <- struct{}{} - }() - - select { - case <-time.After(5 * time.Second): - t.Errorf("unexpected that the reader blocks writer") - case <-done: - } - - if err := rtx.Rollback(); err != nil { - t.Fatal(err) - } -} - -// Ensure that a database cannot open a transaction when it's not open. -func TestDB_Begin_ErrDatabaseNotOpen(t *testing.T) { - var db bolt.DB - if _, err := db.Begin(false); err != bolt.ErrDatabaseNotOpen { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that a read-write transaction can be retrieved. -func TestDB_BeginRW(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } else if tx == nil { - t.Fatal("expected tx") - } - - if tx.DB() != db.DB { - t.Fatal("unexpected tx database") - } else if !tx.Writable() { - t.Fatal("expected writable tx") - } - - if err := tx.Commit(); err != nil { - t.Fatal(err) - } -} - -// Ensure that opening a transaction while the DB is closed returns an error. -func TestDB_BeginRW_Closed(t *testing.T) { - var db bolt.DB - if _, err := db.Begin(true); err != bolt.ErrDatabaseNotOpen { - t.Fatalf("unexpected error: %s", err) - } -} - -func TestDB_Close_PendingTx_RW(t *testing.T) { testDB_Close_PendingTx(t, true) } -func TestDB_Close_PendingTx_RO(t *testing.T) { testDB_Close_PendingTx(t, false) } - -// Ensure that a database cannot close while transactions are open. -func testDB_Close_PendingTx(t *testing.T, writable bool) { - db := MustOpenDB() - defer db.MustClose() - - // Start transaction. - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - - // Open update in separate goroutine. - done := make(chan struct{}) - go func() { - if err := db.Close(); err != nil { - t.Fatal(err) - } - close(done) - }() - - // Ensure database hasn't closed. - time.Sleep(100 * time.Millisecond) - select { - case <-done: - t.Fatal("database closed too early") - default: - } - - // Commit transaction. - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - - // Ensure database closed now. - time.Sleep(100 * time.Millisecond) - select { - case <-done: - default: - t.Fatal("database did not close") - } -} - -// Ensure a database can provide a transactional block. -func TestDB_Update(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte("bat")); err != nil { - t.Fatal(err) - } - if err := b.Delete([]byte("foo")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - if v := b.Get([]byte("foo")); v != nil { - t.Fatalf("expected nil value, got: %v", v) - } - if v := b.Get([]byte("baz")); !bytes.Equal(v, []byte("bat")) { - t.Fatalf("unexpected value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a closed database returns an error while running a transaction block -func TestDB_Update_Closed(t *testing.T) { - var db bolt.DB - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != bolt.ErrDatabaseNotOpen { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure a panic occurs while trying to commit a managed transaction. -func TestDB_Update_ManualCommit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var panicked bool - if err := db.Update(func(tx *bolt.Tx) error { - func() { - defer func() { - if r := recover(); r != nil { - panicked = true - } - }() - - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - }() - return nil - }); err != nil { - t.Fatal(err) - } else if !panicked { - t.Fatal("expected panic") - } -} - -// Ensure a panic occurs while trying to rollback a managed transaction. -func TestDB_Update_ManualRollback(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var panicked bool - if err := db.Update(func(tx *bolt.Tx) error { - func() { - defer func() { - if r := recover(); r != nil { - panicked = true - } - }() - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - }() - return nil - }); err != nil { - t.Fatal(err) - } else if !panicked { - t.Fatal("expected panic") - } -} - -// Ensure a panic occurs while trying to commit a managed transaction. -func TestDB_View_ManualCommit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var panicked bool - if err := db.View(func(tx *bolt.Tx) error { - func() { - defer func() { - if r := recover(); r != nil { - panicked = true - } - }() - - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - }() - return nil - }); err != nil { - t.Fatal(err) - } else if !panicked { - t.Fatal("expected panic") - } -} - -// Ensure a panic occurs while trying to rollback a managed transaction. -func TestDB_View_ManualRollback(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var panicked bool - if err := db.View(func(tx *bolt.Tx) error { - func() { - defer func() { - if r := recover(); r != nil { - panicked = true - } - }() - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - }() - return nil - }); err != nil { - t.Fatal(err) - } else if !panicked { - t.Fatal("expected panic") - } -} - -// Ensure a write transaction that panics does not hold open locks. -func TestDB_Update_Panic(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - // Panic during update but recover. - func() { - defer func() { - if r := recover(); r != nil { - t.Log("recover: update", r) - } - }() - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - panic("omg") - }); err != nil { - t.Fatal(err) - } - }() - - // Verify we can update again. - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Verify that our change persisted. - if err := db.Update(func(tx *bolt.Tx) error { - if tx.Bucket([]byte("widgets")) == nil { - t.Fatal("expected bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure a database can return an error through a read-only transactional block. -func TestDB_View_Error(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.View(func(tx *bolt.Tx) error { - return errors.New("xxx") - }); err == nil || err.Error() != "xxx" { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure a read transaction that panics does not hold open locks. -func TestDB_View_Panic(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Panic during view transaction but recover. - func() { - defer func() { - if r := recover(); r != nil { - t.Log("recover: view", r) - } - }() - - if err := db.View(func(tx *bolt.Tx) error { - if tx.Bucket([]byte("widgets")) == nil { - t.Fatal("expected bucket") - } - panic("omg") - }); err != nil { - t.Fatal(err) - } - }() - - // Verify that we can still use read transactions. - if err := db.View(func(tx *bolt.Tx) error { - if tx.Bucket([]byte("widgets")) == nil { - t.Fatal("expected bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that DB stats can be returned. -func TestDB_Stats(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - t.Fatal(err) - } - - stats := db.Stats() - if stats.TxStats.PageCount != 2 { - t.Fatalf("unexpected TxStats.PageCount: %d", stats.TxStats.PageCount) - } else if stats.FreePageN != 0 { - t.Fatalf("unexpected FreePageN != 0: %d", stats.FreePageN) - } else if stats.PendingPageN != 2 { - t.Fatalf("unexpected PendingPageN != 2: %d", stats.PendingPageN) - } -} - -// Ensure that database pages are in expected order and type. -func TestDB_Consistency(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - t.Fatal(err) - } - - for i := 0; i < 10; i++ { - if err := db.Update(func(tx *bolt.Tx) error { - if err := tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - } - - if err := db.Update(func(tx *bolt.Tx) error { - if p, _ := tx.Page(0); p == nil { - t.Fatal("expected page") - } else if p.Type != "meta" { - t.Fatalf("unexpected page type: %s", p.Type) - } - - if p, _ := tx.Page(1); p == nil { - t.Fatal("expected page") - } else if p.Type != "meta" { - t.Fatalf("unexpected page type: %s", p.Type) - } - - if p, _ := tx.Page(2); p == nil { - t.Fatal("expected page") - } else if p.Type != "free" { - t.Fatalf("unexpected page type: %s", p.Type) - } - - if p, _ := tx.Page(3); p == nil { - t.Fatal("expected page") - } else if p.Type != "free" { - t.Fatalf("unexpected page type: %s", p.Type) - } - - if p, _ := tx.Page(4); p == nil { - t.Fatal("expected page") - } else if p.Type != "leaf" { - t.Fatalf("unexpected page type: %s", p.Type) - } - - if p, _ := tx.Page(5); p == nil { - t.Fatal("expected page") - } else if p.Type != "freelist" { - t.Fatalf("unexpected page type: %s", p.Type) - } - - if p, _ := tx.Page(6); p != nil { - t.Fatal("unexpected page") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that DB stats can be subtracted from one another. -func TestDBStats_Sub(t *testing.T) { - var a, b bolt.Stats - a.TxStats.PageCount = 3 - a.FreePageN = 4 - b.TxStats.PageCount = 10 - b.FreePageN = 14 - diff := b.Sub(&a) - if diff.TxStats.PageCount != 7 { - t.Fatalf("unexpected TxStats.PageCount: %d", diff.TxStats.PageCount) - } - - // free page stats are copied from the receiver and not subtracted - if diff.FreePageN != 14 { - t.Fatalf("unexpected FreePageN: %d", diff.FreePageN) - } -} - -// Ensure two functions can perform updates in a single batch. -func TestDB_Batch(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Iterate over multiple updates in separate goroutines. - n := 2 - ch := make(chan error) - for i := 0; i < n; i++ { - go func(i int) { - ch <- db.Batch(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) - }) - }(i) - } - - // Check all responses to make sure there's no error. - for i := 0; i < n; i++ { - if err := <-ch; err != nil { - t.Fatal(err) - } - } - - // Ensure data is correct. - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for i := 0; i < n; i++ { - if v := b.Get(u64tob(uint64(i))); v == nil { - t.Errorf("key not found: %d", i) - } - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -func TestDB_Batch_Panic(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var sentinel int - var bork = &sentinel - var problem interface{} - var err error - - // Execute a function inside a batch that panics. - func() { - defer func() { - if p := recover(); p != nil { - problem = p - } - }() - err = db.Batch(func(tx *bolt.Tx) error { - panic(bork) - }) - }() - - // Verify there is no error. - if g, e := err, error(nil); g != e { - t.Fatalf("wrong error: %v != %v", g, e) - } - // Verify the panic was captured. - if g, e := problem, bork; g != e { - t.Fatalf("wrong error: %v != %v", g, e) - } -} - -func TestDB_BatchFull(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - t.Fatal(err) - } - - const size = 3 - // buffered so we never leak goroutines - ch := make(chan error, size) - put := func(i int) { - ch <- db.Batch(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) - }) - } - - db.MaxBatchSize = size - // high enough to never trigger here - db.MaxBatchDelay = 1 * time.Hour - - go put(1) - go put(2) - - // Give the batch a chance to exhibit bugs. - time.Sleep(10 * time.Millisecond) - - // not triggered yet - select { - case <-ch: - t.Fatalf("batch triggered too early") - default: - } - - go put(3) - - // Check all responses to make sure there's no error. - for i := 0; i < size; i++ { - if err := <-ch; err != nil { - t.Fatal(err) - } - } - - // Ensure data is correct. - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for i := 1; i <= size; i++ { - if v := b.Get(u64tob(uint64(i))); v == nil { - t.Errorf("key not found: %d", i) - } - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -func TestDB_BatchTime(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - t.Fatal(err) - } - - const size = 1 - // buffered so we never leak goroutines - ch := make(chan error, size) - put := func(i int) { - ch <- db.Batch(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("widgets")).Put(u64tob(uint64(i)), []byte{}) - }) - } - - db.MaxBatchSize = 1000 - db.MaxBatchDelay = 0 - - go put(1) - - // Batch must trigger by time alone. - - // Check all responses to make sure there's no error. - for i := 0; i < size; i++ { - if err := <-ch; err != nil { - t.Fatal(err) - } - } - - // Ensure data is correct. - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("widgets")) - for i := 1; i <= size; i++ { - if v := b.Get(u64tob(uint64(i))); v == nil { - t.Errorf("key not found: %d", i) - } - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -func ExampleDB_Update() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Execute several commands within a read-write transaction. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - return err - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - return err - } - return nil - }); err != nil { - log.Fatal(err) - } - - // Read the value back from a separate read-only transaction. - if err := db.View(func(tx *bolt.Tx) error { - value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value of 'foo' is: %s\n", value) - return nil - }); err != nil { - log.Fatal(err) - } - - // Close database to release the file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // The value of 'foo' is: bar -} - -func ExampleDB_View() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Insert data into a bucket. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("people")) - if err != nil { - return err - } - if err := b.Put([]byte("john"), []byte("doe")); err != nil { - return err - } - if err := b.Put([]byte("susy"), []byte("que")); err != nil { - return err - } - return nil - }); err != nil { - log.Fatal(err) - } - - // Access data from within a read-only transactional block. - if err := db.View(func(tx *bolt.Tx) error { - v := tx.Bucket([]byte("people")).Get([]byte("john")) - fmt.Printf("John's last name is %s.\n", v) - return nil - }); err != nil { - log.Fatal(err) - } - - // Close database to release the file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // John's last name is doe. -} - -func ExampleDB_Begin_ReadOnly() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Create a bucket using a read-write transaction. - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - log.Fatal(err) - } - - // Create several keys in a transaction. - tx, err := db.Begin(true) - if err != nil { - log.Fatal(err) - } - b := tx.Bucket([]byte("widgets")) - if err := b.Put([]byte("john"), []byte("blue")); err != nil { - log.Fatal(err) - } - if err := b.Put([]byte("abby"), []byte("red")); err != nil { - log.Fatal(err) - } - if err := b.Put([]byte("zephyr"), []byte("purple")); err != nil { - log.Fatal(err) - } - if err := tx.Commit(); err != nil { - log.Fatal(err) - } - - // Iterate over the values in sorted key order. - tx, err = db.Begin(false) - if err != nil { - log.Fatal(err) - } - c := tx.Bucket([]byte("widgets")).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - fmt.Printf("%s likes %s\n", k, v) - } - - if err := tx.Rollback(); err != nil { - log.Fatal(err) - } - - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // abby likes red - // john likes blue - // zephyr likes purple -} - -func BenchmarkDBBatchAutomatic(b *testing.B) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("bench")) - return err - }); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - start := make(chan struct{}) - var wg sync.WaitGroup - - for round := 0; round < 1000; round++ { - wg.Add(1) - - go func(id uint32) { - defer wg.Done() - <-start - - h := fnv.New32a() - buf := make([]byte, 4) - binary.LittleEndian.PutUint32(buf, id) - _, _ = h.Write(buf[:]) - k := h.Sum(nil) - insert := func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("bench")) - return b.Put(k, []byte("filler")) - } - if err := db.Batch(insert); err != nil { - b.Error(err) - return - } - }(uint32(round)) - } - close(start) - wg.Wait() - } - - b.StopTimer() - validateBatchBench(b, db) -} - -func BenchmarkDBBatchSingle(b *testing.B) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("bench")) - return err - }); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - start := make(chan struct{}) - var wg sync.WaitGroup - - for round := 0; round < 1000; round++ { - wg.Add(1) - go func(id uint32) { - defer wg.Done() - <-start - - h := fnv.New32a() - buf := make([]byte, 4) - binary.LittleEndian.PutUint32(buf, id) - _, _ = h.Write(buf[:]) - k := h.Sum(nil) - insert := func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("bench")) - return b.Put(k, []byte("filler")) - } - if err := db.Update(insert); err != nil { - b.Error(err) - return - } - }(uint32(round)) - } - close(start) - wg.Wait() - } - - b.StopTimer() - validateBatchBench(b, db) -} - -func BenchmarkDBBatchManual10x100(b *testing.B) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("bench")) - return err - }); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - start := make(chan struct{}) - var wg sync.WaitGroup - - for major := 0; major < 10; major++ { - wg.Add(1) - go func(id uint32) { - defer wg.Done() - <-start - - insert100 := func(tx *bolt.Tx) error { - h := fnv.New32a() - buf := make([]byte, 4) - for minor := uint32(0); minor < 100; minor++ { - binary.LittleEndian.PutUint32(buf, uint32(id*100+minor)) - h.Reset() - _, _ = h.Write(buf[:]) - k := h.Sum(nil) - b := tx.Bucket([]byte("bench")) - if err := b.Put(k, []byte("filler")); err != nil { - return err - } - } - return nil - } - if err := db.Update(insert100); err != nil { - b.Fatal(err) - } - }(uint32(major)) - } - close(start) - wg.Wait() - } - - b.StopTimer() - validateBatchBench(b, db) -} - -func validateBatchBench(b *testing.B, db *DB) { - var rollback = errors.New("sentinel error to cause rollback") - validate := func(tx *bolt.Tx) error { - bucket := tx.Bucket([]byte("bench")) - h := fnv.New32a() - buf := make([]byte, 4) - for id := uint32(0); id < 1000; id++ { - binary.LittleEndian.PutUint32(buf, id) - h.Reset() - _, _ = h.Write(buf[:]) - k := h.Sum(nil) - v := bucket.Get(k) - if v == nil { - b.Errorf("not found id=%d key=%x", id, k) - continue - } - if g, e := v, []byte("filler"); !bytes.Equal(g, e) { - b.Errorf("bad value for id=%d key=%x: %s != %q", id, k, g, e) - } - if err := bucket.Delete(k); err != nil { - return err - } - } - // should be empty now - c := bucket.Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - b.Errorf("unexpected key: %x = %q", k, v) - } - return rollback - } - if err := db.Update(validate); err != nil && err != rollback { - b.Error(err) - } -} - -// DB is a test wrapper for bolt.DB. -type DB struct { - *bolt.DB -} - -// MustOpenDB returns a new, open DB at a temporary location. -func MustOpenDB() *DB { - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - panic(err) - } - return &DB{db} -} - -// Close closes the database and deletes the underlying file. -func (db *DB) Close() error { - // Log statistics. - if *statsFlag { - db.PrintStats() - } - - // Check database consistency after every test. - db.MustCheck() - - // Close database and remove file. - defer os.Remove(db.Path()) - return db.DB.Close() -} - -// MustClose closes the database and deletes the underlying file. Panic on error. -func (db *DB) MustClose() { - if err := db.Close(); err != nil { - panic(err) - } -} - -// PrintStats prints the database stats -func (db *DB) PrintStats() { - var stats = db.Stats() - fmt.Printf("[db] %-20s %-20s %-20s\n", - fmt.Sprintf("pg(%d/%d)", stats.TxStats.PageCount, stats.TxStats.PageAlloc), - fmt.Sprintf("cur(%d)", stats.TxStats.CursorCount), - fmt.Sprintf("node(%d/%d)", stats.TxStats.NodeCount, stats.TxStats.NodeDeref), - ) - fmt.Printf(" %-20s %-20s %-20s\n", - fmt.Sprintf("rebal(%d/%v)", stats.TxStats.Rebalance, truncDuration(stats.TxStats.RebalanceTime)), - fmt.Sprintf("spill(%d/%v)", stats.TxStats.Spill, truncDuration(stats.TxStats.SpillTime)), - fmt.Sprintf("w(%d/%v)", stats.TxStats.Write, truncDuration(stats.TxStats.WriteTime)), - ) -} - -// MustCheck runs a consistency check on the database and panics if any errors are found. -func (db *DB) MustCheck() { - if err := db.Update(func(tx *bolt.Tx) error { - // Collect all the errors. - var errors []error - for err := range tx.Check() { - errors = append(errors, err) - if len(errors) > 10 { - break - } - } - - // If errors occurred, copy the DB and print the errors. - if len(errors) > 0 { - var path = tempfile() - if err := tx.CopyFile(path, 0600); err != nil { - panic(err) - } - - // Print errors. - fmt.Print("\n\n") - fmt.Printf("consistency check failed (%d errors)\n", len(errors)) - for _, err := range errors { - fmt.Println(err) - } - fmt.Println("") - fmt.Println("db saved to:") - fmt.Println(path) - fmt.Print("\n\n") - os.Exit(-1) - } - - return nil - }); err != nil && err != bolt.ErrDatabaseNotOpen { - panic(err) - } -} - -// CopyTempFile copies a database to a temporary file. -func (db *DB) CopyTempFile() { - path := tempfile() - if err := db.View(func(tx *bolt.Tx) error { - return tx.CopyFile(path, 0600) - }); err != nil { - panic(err) - } - fmt.Println("db copied to: ", path) -} - -// tempfile returns a temporary file path. -func tempfile() string { - f, err := ioutil.TempFile("", "bolt-") - if err != nil { - panic(err) - } - if err := f.Close(); err != nil { - panic(err) - } - if err := os.Remove(f.Name()); err != nil { - panic(err) - } - return f.Name() -} - -// mustContainKeys checks that a bucket contains a given set of keys. -func mustContainKeys(b *bolt.Bucket, m map[string]string) { - found := make(map[string]string) - if err := b.ForEach(func(k, _ []byte) error { - found[string(k)] = "" - return nil - }); err != nil { - panic(err) - } - - // Check for keys found in bucket that shouldn't be there. - var keys []string - for k, _ := range found { - if _, ok := m[string(k)]; !ok { - keys = append(keys, k) - } - } - if len(keys) > 0 { - sort.Strings(keys) - panic(fmt.Sprintf("keys found(%d): %s", len(keys), strings.Join(keys, ","))) - } - - // Check for keys not found in bucket that should be there. - for k, _ := range m { - if _, ok := found[string(k)]; !ok { - keys = append(keys, k) - } - } - if len(keys) > 0 { - sort.Strings(keys) - panic(fmt.Sprintf("keys not found(%d): %s", len(keys), strings.Join(keys, ","))) - } -} - -func trunc(b []byte, length int) []byte { - if length < len(b) { - return b[:length] - } - return b -} - -func truncDuration(d time.Duration) string { - return regexp.MustCompile(`^(\d+)(\.\d+)`).ReplaceAllString(d.String(), "$1") -} - -func fileSize(path string) int64 { - fi, err := os.Stat(path) - if err != nil { - return 0 - } - return fi.Size() -} - -func warn(v ...interface{}) { fmt.Fprintln(os.Stderr, v...) } -func warnf(msg string, v ...interface{}) { fmt.Fprintf(os.Stderr, msg+"\n", v...) } - -// u64tob converts a uint64 into an 8-byte slice. -func u64tob(v uint64) []byte { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, v) - return b -} - -// btou64 converts an 8-byte slice into an uint64. -func btou64(b []byte) uint64 { return binary.BigEndian.Uint64(b) } diff --git a/vendor/github.com/boltdb/bolt/errors.go b/vendor/github.com/boltdb/bolt/errors.go index 6883786d..a3620a3e 100644 --- a/vendor/github.com/boltdb/bolt/errors.go +++ b/vendor/github.com/boltdb/bolt/errors.go @@ -12,7 +12,8 @@ var ( // already open. ErrDatabaseOpen = errors.New("database already open") - // ErrInvalid is returned when a data file is not a Bolt-formatted database. + // ErrInvalid is returned when both meta pages on a database are invalid. + // This typically occurs when a file is not a bolt database. ErrInvalid = errors.New("invalid database") // ErrVersionMismatch is returned when the data file was created with a diff --git a/vendor/github.com/boltdb/bolt/freelist.go b/vendor/github.com/boltdb/bolt/freelist.go index 0161948f..aba48f58 100644 --- a/vendor/github.com/boltdb/bolt/freelist.go +++ b/vendor/github.com/boltdb/bolt/freelist.go @@ -24,7 +24,12 @@ func newFreelist() *freelist { // size returns the size of the page after serialization. func (f *freelist) size() int { - return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count()) + n := f.count() + if n >= 0xFFFF { + // The first element will be used to store the count. See freelist.write. + n++ + } + return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n) } // count returns count of pages on the freelist @@ -46,16 +51,15 @@ func (f *freelist) pending_count() int { return count } -// all returns a list of all free ids and all pending ids in one sorted list. -func (f *freelist) all() []pgid { - m := make(pgids, 0) - +// copyall copies into dst a list of all free ids and all pending ids in one sorted list. +// f.count returns the minimum length required for dst. +func (f *freelist) copyall(dst []pgid) { + m := make(pgids, 0, f.pending_count()) for _, list := range f.pending { m = append(m, list...) } - sort.Sort(m) - return pgids(f.ids).merge(m) + mergepgids(dst, f.ids, m) } // allocate returns the starting page id of a contiguous list of pages of a given size. @@ -166,12 +170,16 @@ func (f *freelist) read(p *page) { } // Copy the list of page ids from the freelist. - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] - f.ids = make([]pgid, len(ids)) - copy(f.ids, ids) + if count == 0 { + f.ids = nil + } else { + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + f.ids = make([]pgid, len(ids)) + copy(f.ids, ids) - // Make sure they're sorted. - sort.Sort(pgids(f.ids)) + // Make sure they're sorted. + sort.Sort(pgids(f.ids)) + } // Rebuild the page cache. f.reindex() @@ -182,20 +190,22 @@ func (f *freelist) read(p *page) { // become free. func (f *freelist) write(p *page) error { // Combine the old free pgids and pgids waiting on an open transaction. - ids := f.all() // Update the header flag. p.flags |= freelistPageFlag // The page.count can only hold up to 64k elements so if we overflow that // number then we handle it by putting the size in the first element. - if len(ids) < 0xFFFF { - p.count = uint16(len(ids)) - copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids) + lenids := f.count() + if lenids == 0 { + p.count = uint16(lenids) + } else if lenids < 0xFFFF { + p.count = uint16(lenids) + f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:]) } else { p.count = 0xFFFF - ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids)) - copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids) + ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids) + f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:]) } return nil @@ -230,7 +240,7 @@ func (f *freelist) reload(p *page) { // reindex rebuilds the free cache based on available and pending free lists. func (f *freelist) reindex() { - f.cache = make(map[pgid]bool) + f.cache = make(map[pgid]bool, len(f.ids)) for _, id := range f.ids { f.cache[id] = true } diff --git a/vendor/github.com/boltdb/bolt/freelist_test.go b/vendor/github.com/boltdb/bolt/freelist_test.go deleted file mode 100644 index 4e9b3a8d..00000000 --- a/vendor/github.com/boltdb/bolt/freelist_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package bolt - -import ( - "math/rand" - "reflect" - "sort" - "testing" - "unsafe" -) - -// Ensure that a page is added to a transaction's freelist. -func TestFreelist_free(t *testing.T) { - f := newFreelist() - f.free(100, &page{id: 12}) - if !reflect.DeepEqual([]pgid{12}, f.pending[100]) { - t.Fatalf("exp=%v; got=%v", []pgid{12}, f.pending[100]) - } -} - -// Ensure that a page and its overflow is added to a transaction's freelist. -func TestFreelist_free_overflow(t *testing.T) { - f := newFreelist() - f.free(100, &page{id: 12, overflow: 3}) - if exp := []pgid{12, 13, 14, 15}; !reflect.DeepEqual(exp, f.pending[100]) { - t.Fatalf("exp=%v; got=%v", exp, f.pending[100]) - } -} - -// Ensure that a transaction's free pages can be released. -func TestFreelist_release(t *testing.T) { - f := newFreelist() - f.free(100, &page{id: 12, overflow: 1}) - f.free(100, &page{id: 9}) - f.free(102, &page{id: 39}) - f.release(100) - f.release(101) - if exp := []pgid{9, 12, 13}; !reflect.DeepEqual(exp, f.ids) { - t.Fatalf("exp=%v; got=%v", exp, f.ids) - } - - f.release(102) - if exp := []pgid{9, 12, 13, 39}; !reflect.DeepEqual(exp, f.ids) { - t.Fatalf("exp=%v; got=%v", exp, f.ids) - } -} - -// Ensure that a freelist can find contiguous blocks of pages. -func TestFreelist_allocate(t *testing.T) { - f := &freelist{ids: []pgid{3, 4, 5, 6, 7, 9, 12, 13, 18}} - if id := int(f.allocate(3)); id != 3 { - t.Fatalf("exp=3; got=%v", id) - } - if id := int(f.allocate(1)); id != 6 { - t.Fatalf("exp=6; got=%v", id) - } - if id := int(f.allocate(3)); id != 0 { - t.Fatalf("exp=0; got=%v", id) - } - if id := int(f.allocate(2)); id != 12 { - t.Fatalf("exp=12; got=%v", id) - } - if id := int(f.allocate(1)); id != 7 { - t.Fatalf("exp=7; got=%v", id) - } - if id := int(f.allocate(0)); id != 0 { - t.Fatalf("exp=0; got=%v", id) - } - if id := int(f.allocate(0)); id != 0 { - t.Fatalf("exp=0; got=%v", id) - } - if exp := []pgid{9, 18}; !reflect.DeepEqual(exp, f.ids) { - t.Fatalf("exp=%v; got=%v", exp, f.ids) - } - - if id := int(f.allocate(1)); id != 9 { - t.Fatalf("exp=9; got=%v", id) - } - if id := int(f.allocate(1)); id != 18 { - t.Fatalf("exp=18; got=%v", id) - } - if id := int(f.allocate(1)); id != 0 { - t.Fatalf("exp=0; got=%v", id) - } - if exp := []pgid{}; !reflect.DeepEqual(exp, f.ids) { - t.Fatalf("exp=%v; got=%v", exp, f.ids) - } -} - -// Ensure that a freelist can deserialize from a freelist page. -func TestFreelist_read(t *testing.T) { - // Create a page. - var buf [4096]byte - page := (*page)(unsafe.Pointer(&buf[0])) - page.flags = freelistPageFlag - page.count = 2 - - // Insert 2 page ids. - ids := (*[3]pgid)(unsafe.Pointer(&page.ptr)) - ids[0] = 23 - ids[1] = 50 - - // Deserialize page into a freelist. - f := newFreelist() - f.read(page) - - // Ensure that there are two page ids in the freelist. - if exp := []pgid{23, 50}; !reflect.DeepEqual(exp, f.ids) { - t.Fatalf("exp=%v; got=%v", exp, f.ids) - } -} - -// Ensure that a freelist can serialize into a freelist page. -func TestFreelist_write(t *testing.T) { - // Create a freelist and write it to a page. - var buf [4096]byte - f := &freelist{ids: []pgid{12, 39}, pending: make(map[txid][]pgid)} - f.pending[100] = []pgid{28, 11} - f.pending[101] = []pgid{3} - p := (*page)(unsafe.Pointer(&buf[0])) - if err := f.write(p); err != nil { - t.Fatal(err) - } - - // Read the page back out. - f2 := newFreelist() - f2.read(p) - - // Ensure that the freelist is correct. - // All pages should be present and in reverse order. - if exp := []pgid{3, 11, 12, 28, 39}; !reflect.DeepEqual(exp, f2.ids) { - t.Fatalf("exp=%v; got=%v", exp, f2.ids) - } -} - -func Benchmark_FreelistRelease10K(b *testing.B) { benchmark_FreelistRelease(b, 10000) } -func Benchmark_FreelistRelease100K(b *testing.B) { benchmark_FreelistRelease(b, 100000) } -func Benchmark_FreelistRelease1000K(b *testing.B) { benchmark_FreelistRelease(b, 1000000) } -func Benchmark_FreelistRelease10000K(b *testing.B) { benchmark_FreelistRelease(b, 10000000) } - -func benchmark_FreelistRelease(b *testing.B, size int) { - ids := randomPgids(size) - pending := randomPgids(len(ids) / 400) - b.ResetTimer() - for i := 0; i < b.N; i++ { - f := &freelist{ids: ids, pending: map[txid][]pgid{1: pending}} - f.release(1) - } -} - -func randomPgids(n int) []pgid { - rand.Seed(42) - pgids := make(pgids, n) - for i := range pgids { - pgids[i] = pgid(rand.Int63()) - } - sort.Sort(pgids) - return pgids -} diff --git a/vendor/github.com/boltdb/bolt/node.go b/vendor/github.com/boltdb/bolt/node.go index c9fb21c7..159318b2 100644 --- a/vendor/github.com/boltdb/bolt/node.go +++ b/vendor/github.com/boltdb/bolt/node.go @@ -201,6 +201,11 @@ func (n *node) write(p *page) { } p.count = uint16(len(n.inodes)) + // Stop here if there are no items to write. + if p.count == 0 { + return + } + // Loop over each item and write it to the page. b := (*[maxAllocSize]byte)(unsafe.Pointer(&p.ptr))[n.pageElementSize()*len(n.inodes):] for i, item := range n.inodes { @@ -463,43 +468,6 @@ func (n *node) rebalance() { target = n.prevSibling() } - // If target node has extra nodes then just move one over. - if target.numChildren() > target.minKeys() { - if useNextSibling { - // Reparent and move node. - if child, ok := n.bucket.nodes[target.inodes[0].pgid]; ok { - child.parent.removeChild(child) - child.parent = n - child.parent.children = append(child.parent.children, child) - } - n.inodes = append(n.inodes, target.inodes[0]) - target.inodes = target.inodes[1:] - - // Update target key on parent. - target.parent.put(target.key, target.inodes[0].key, nil, target.pgid, 0) - target.key = target.inodes[0].key - _assert(len(target.key) > 0, "rebalance(1): zero-length node key") - } else { - // Reparent and move node. - if child, ok := n.bucket.nodes[target.inodes[len(target.inodes)-1].pgid]; ok { - child.parent.removeChild(child) - child.parent = n - child.parent.children = append(child.parent.children, child) - } - n.inodes = append(n.inodes, inode{}) - copy(n.inodes[1:], n.inodes) - n.inodes[0] = target.inodes[len(target.inodes)-1] - target.inodes = target.inodes[:len(target.inodes)-1] - } - - // Update parent key for node. - n.parent.put(n.key, n.inodes[0].key, nil, n.pgid, 0) - n.key = n.inodes[0].key - _assert(len(n.key) > 0, "rebalance(2): zero-length node key") - - return - } - // If both this node and the target node are too small then merge them. if useNextSibling { // Reparent all child nodes being moved. diff --git a/vendor/github.com/boltdb/bolt/node_test.go b/vendor/github.com/boltdb/bolt/node_test.go deleted file mode 100644 index fa5d10f9..00000000 --- a/vendor/github.com/boltdb/bolt/node_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package bolt - -import ( - "testing" - "unsafe" -) - -// Ensure that a node can insert a key/value. -func TestNode_put(t *testing.T) { - n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{meta: &meta{pgid: 1}}}} - n.put([]byte("baz"), []byte("baz"), []byte("2"), 0, 0) - n.put([]byte("foo"), []byte("foo"), []byte("0"), 0, 0) - n.put([]byte("bar"), []byte("bar"), []byte("1"), 0, 0) - n.put([]byte("foo"), []byte("foo"), []byte("3"), 0, leafPageFlag) - - if len(n.inodes) != 3 { - t.Fatalf("exp=3; got=%d", len(n.inodes)) - } - if k, v := n.inodes[0].key, n.inodes[0].value; string(k) != "bar" || string(v) != "1" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } - if k, v := n.inodes[1].key, n.inodes[1].value; string(k) != "baz" || string(v) != "2" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } - if k, v := n.inodes[2].key, n.inodes[2].value; string(k) != "foo" || string(v) != "3" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } - if n.inodes[2].flags != uint32(leafPageFlag) { - t.Fatalf("not a leaf: %d", n.inodes[2].flags) - } -} - -// Ensure that a node can deserialize from a leaf page. -func TestNode_read_LeafPage(t *testing.T) { - // Create a page. - var buf [4096]byte - page := (*page)(unsafe.Pointer(&buf[0])) - page.flags = leafPageFlag - page.count = 2 - - // Insert 2 elements at the beginning. sizeof(leafPageElement) == 16 - nodes := (*[3]leafPageElement)(unsafe.Pointer(&page.ptr)) - nodes[0] = leafPageElement{flags: 0, pos: 32, ksize: 3, vsize: 4} // pos = sizeof(leafPageElement) * 2 - nodes[1] = leafPageElement{flags: 0, pos: 23, ksize: 10, vsize: 3} // pos = sizeof(leafPageElement) + 3 + 4 - - // Write data for the nodes at the end. - data := (*[4096]byte)(unsafe.Pointer(&nodes[2])) - copy(data[:], []byte("barfooz")) - copy(data[7:], []byte("helloworldbye")) - - // Deserialize page into a leaf. - n := &node{} - n.read(page) - - // Check that there are two inodes with correct data. - if !n.isLeaf { - t.Fatal("expected leaf") - } - if len(n.inodes) != 2 { - t.Fatalf("exp=2; got=%d", len(n.inodes)) - } - if k, v := n.inodes[0].key, n.inodes[0].value; string(k) != "bar" || string(v) != "fooz" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } - if k, v := n.inodes[1].key, n.inodes[1].value; string(k) != "helloworld" || string(v) != "bye" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } -} - -// Ensure that a node can serialize into a leaf page. -func TestNode_write_LeafPage(t *testing.T) { - // Create a node. - n := &node{isLeaf: true, inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} - n.put([]byte("susy"), []byte("susy"), []byte("que"), 0, 0) - n.put([]byte("ricki"), []byte("ricki"), []byte("lake"), 0, 0) - n.put([]byte("john"), []byte("john"), []byte("johnson"), 0, 0) - - // Write it to a page. - var buf [4096]byte - p := (*page)(unsafe.Pointer(&buf[0])) - n.write(p) - - // Read the page back in. - n2 := &node{} - n2.read(p) - - // Check that the two pages are the same. - if len(n2.inodes) != 3 { - t.Fatalf("exp=3; got=%d", len(n2.inodes)) - } - if k, v := n2.inodes[0].key, n2.inodes[0].value; string(k) != "john" || string(v) != "johnson" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } - if k, v := n2.inodes[1].key, n2.inodes[1].value; string(k) != "ricki" || string(v) != "lake" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } - if k, v := n2.inodes[2].key, n2.inodes[2].value; string(k) != "susy" || string(v) != "que" { - t.Fatalf("exp=; got=<%s,%s>", k, v) - } -} - -// Ensure that a node can split into appropriate subgroups. -func TestNode_split(t *testing.T) { - // Create a node. - n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} - n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000004"), []byte("00000004"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000005"), []byte("00000005"), []byte("0123456701234567"), 0, 0) - - // Split between 2 & 3. - n.split(100) - - var parent = n.parent - if len(parent.children) != 2 { - t.Fatalf("exp=2; got=%d", len(parent.children)) - } - if len(parent.children[0].inodes) != 2 { - t.Fatalf("exp=2; got=%d", len(parent.children[0].inodes)) - } - if len(parent.children[1].inodes) != 3 { - t.Fatalf("exp=3; got=%d", len(parent.children[1].inodes)) - } -} - -// Ensure that a page with the minimum number of inodes just returns a single node. -func TestNode_split_MinKeys(t *testing.T) { - // Create a node. - n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} - n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) - - // Split. - n.split(20) - if n.parent != nil { - t.Fatalf("expected nil parent") - } -} - -// Ensure that a node that has keys that all fit on a page just returns one leaf. -func TestNode_split_SinglePage(t *testing.T) { - // Create a node. - n := &node{inodes: make(inodes, 0), bucket: &Bucket{tx: &Tx{db: &DB{}, meta: &meta{pgid: 1}}}} - n.put([]byte("00000001"), []byte("00000001"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000002"), []byte("00000002"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000003"), []byte("00000003"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000004"), []byte("00000004"), []byte("0123456701234567"), 0, 0) - n.put([]byte("00000005"), []byte("00000005"), []byte("0123456701234567"), 0, 0) - - // Split. - n.split(4096) - if n.parent != nil { - t.Fatalf("expected nil parent") - } -} diff --git a/vendor/github.com/boltdb/bolt/page.go b/vendor/github.com/boltdb/bolt/page.go index 818aa1b1..cde403ae 100644 --- a/vendor/github.com/boltdb/bolt/page.go +++ b/vendor/github.com/boltdb/bolt/page.go @@ -62,6 +62,9 @@ func (p *page) leafPageElement(index uint16) *leafPageElement { // leafPageElements retrieves a list of leaf nodes. func (p *page) leafPageElements() []leafPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] } @@ -72,6 +75,9 @@ func (p *page) branchPageElement(index uint16) *branchPageElement { // branchPageElements retrieves a list of branch nodes. func (p *page) branchPageElements() []branchPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] } @@ -111,13 +117,13 @@ type leafPageElement struct { // key returns a byte slice of the node key. func (n *leafPageElement) key() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize] + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize] } // value returns a byte slice of the node value. func (n *leafPageElement) value() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) - return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize] + return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize] } // PageInfo represents human readable information about a page. @@ -139,12 +145,33 @@ func (a pgids) merge(b pgids) pgids { // Return the opposite slice if one is nil. if len(a) == 0 { return b - } else if len(b) == 0 { + } + if len(b) == 0 { return a } + merged := make(pgids, len(a)+len(b)) + mergepgids(merged, a, b) + return merged +} - // Create a list to hold all elements from both lists. - merged := make(pgids, 0, len(a)+len(b)) +// mergepgids copies the sorted union of a and b into dst. +// If dst is too small, it panics. +func mergepgids(dst, a, b pgids) { + if len(dst) < len(a)+len(b) { + panic(fmt.Errorf("mergepgids bad len %d < %d + %d", len(dst), len(a), len(b))) + } + // Copy in the opposite slice if one is nil. + if len(a) == 0 { + copy(dst, b) + return + } + if len(b) == 0 { + copy(dst, a) + return + } + + // Merged will hold all elements from both lists. + merged := dst[:0] // Assign lead to the slice with a lower starting value, follow to the higher value. lead, follow := a, b @@ -166,7 +193,5 @@ func (a pgids) merge(b pgids) pgids { } // Append what's left in follow. - merged = append(merged, follow...) - - return merged + _ = append(merged, follow...) } diff --git a/vendor/github.com/boltdb/bolt/page_test.go b/vendor/github.com/boltdb/bolt/page_test.go deleted file mode 100644 index 59f4a30e..00000000 --- a/vendor/github.com/boltdb/bolt/page_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package bolt - -import ( - "reflect" - "sort" - "testing" - "testing/quick" -) - -// Ensure that the page type can be returned in human readable format. -func TestPage_typ(t *testing.T) { - if typ := (&page{flags: branchPageFlag}).typ(); typ != "branch" { - t.Fatalf("exp=branch; got=%v", typ) - } - if typ := (&page{flags: leafPageFlag}).typ(); typ != "leaf" { - t.Fatalf("exp=leaf; got=%v", typ) - } - if typ := (&page{flags: metaPageFlag}).typ(); typ != "meta" { - t.Fatalf("exp=meta; got=%v", typ) - } - if typ := (&page{flags: freelistPageFlag}).typ(); typ != "freelist" { - t.Fatalf("exp=freelist; got=%v", typ) - } - if typ := (&page{flags: 20000}).typ(); typ != "unknown<4e20>" { - t.Fatalf("exp=unknown<4e20>; got=%v", typ) - } -} - -// Ensure that the hexdump debugging function doesn't blow up. -func TestPage_dump(t *testing.T) { - (&page{id: 256}).hexdump(16) -} - -func TestPgids_merge(t *testing.T) { - a := pgids{4, 5, 6, 10, 11, 12, 13, 27} - b := pgids{1, 3, 8, 9, 25, 30} - c := a.merge(b) - if !reflect.DeepEqual(c, pgids{1, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30}) { - t.Errorf("mismatch: %v", c) - } - - a = pgids{4, 5, 6, 10, 11, 12, 13, 27, 35, 36} - b = pgids{8, 9, 25, 30} - c = a.merge(b) - if !reflect.DeepEqual(c, pgids{4, 5, 6, 8, 9, 10, 11, 12, 13, 25, 27, 30, 35, 36}) { - t.Errorf("mismatch: %v", c) - } -} - -func TestPgids_merge_quick(t *testing.T) { - if err := quick.Check(func(a, b pgids) bool { - // Sort incoming lists. - sort.Sort(a) - sort.Sort(b) - - // Merge the two lists together. - got := a.merge(b) - - // The expected value should be the two lists combined and sorted. - exp := append(a, b...) - sort.Sort(exp) - - if !reflect.DeepEqual(exp, got) { - t.Errorf("\nexp=%+v\ngot=%+v\n", exp, got) - return false - } - - return true - }, nil); err != nil { - t.Fatal(err) - } -} diff --git a/vendor/github.com/boltdb/bolt/quick_test.go b/vendor/github.com/boltdb/bolt/quick_test.go deleted file mode 100644 index 4da58177..00000000 --- a/vendor/github.com/boltdb/bolt/quick_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package bolt_test - -import ( - "bytes" - "flag" - "fmt" - "math/rand" - "os" - "reflect" - "testing/quick" - "time" -) - -// testing/quick defaults to 5 iterations and a random seed. -// You can override these settings from the command line: -// -// -quick.count The number of iterations to perform. -// -quick.seed The seed to use for randomizing. -// -quick.maxitems The maximum number of items to insert into a DB. -// -quick.maxksize The maximum size of a key. -// -quick.maxvsize The maximum size of a value. -// - -var qcount, qseed, qmaxitems, qmaxksize, qmaxvsize int - -func init() { - flag.IntVar(&qcount, "quick.count", 5, "") - flag.IntVar(&qseed, "quick.seed", int(time.Now().UnixNano())%100000, "") - flag.IntVar(&qmaxitems, "quick.maxitems", 1000, "") - flag.IntVar(&qmaxksize, "quick.maxksize", 1024, "") - flag.IntVar(&qmaxvsize, "quick.maxvsize", 1024, "") - flag.Parse() - fmt.Fprintln(os.Stderr, "seed:", qseed) - fmt.Fprintf(os.Stderr, "quick settings: count=%v, items=%v, ksize=%v, vsize=%v\n", qcount, qmaxitems, qmaxksize, qmaxvsize) -} - -func qconfig() *quick.Config { - return &quick.Config{ - MaxCount: qcount, - Rand: rand.New(rand.NewSource(int64(qseed))), - } -} - -type testdata []testdataitem - -func (t testdata) Len() int { return len(t) } -func (t testdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == -1 } - -func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { - n := rand.Intn(qmaxitems-1) + 1 - items := make(testdata, n) - for i := 0; i < n; i++ { - item := &items[i] - item.Key = randByteSlice(rand, 1, qmaxksize) - item.Value = randByteSlice(rand, 0, qmaxvsize) - } - return reflect.ValueOf(items) -} - -type revtestdata []testdataitem - -func (t revtestdata) Len() int { return len(t) } -func (t revtestdata) Swap(i, j int) { t[i], t[j] = t[j], t[i] } -func (t revtestdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) == 1 } - -type testdataitem struct { - Key []byte - Value []byte -} - -func randByteSlice(rand *rand.Rand, minSize, maxSize int) []byte { - n := rand.Intn(maxSize-minSize) + minSize - b := make([]byte, n) - for i := 0; i < n; i++ { - b[i] = byte(rand.Intn(255)) - } - return b -} diff --git a/vendor/github.com/boltdb/bolt/simulation_test.go b/vendor/github.com/boltdb/bolt/simulation_test.go deleted file mode 100644 index 38310165..00000000 --- a/vendor/github.com/boltdb/bolt/simulation_test.go +++ /dev/null @@ -1,329 +0,0 @@ -package bolt_test - -import ( - "bytes" - "fmt" - "math/rand" - "sync" - "testing" - - "github.com/boltdb/bolt" -) - -func TestSimulate_1op_1p(t *testing.T) { testSimulate(t, 1, 1) } -func TestSimulate_10op_1p(t *testing.T) { testSimulate(t, 10, 1) } -func TestSimulate_100op_1p(t *testing.T) { testSimulate(t, 100, 1) } -func TestSimulate_1000op_1p(t *testing.T) { testSimulate(t, 1000, 1) } -func TestSimulate_10000op_1p(t *testing.T) { testSimulate(t, 10000, 1) } - -func TestSimulate_10op_10p(t *testing.T) { testSimulate(t, 10, 10) } -func TestSimulate_100op_10p(t *testing.T) { testSimulate(t, 100, 10) } -func TestSimulate_1000op_10p(t *testing.T) { testSimulate(t, 1000, 10) } -func TestSimulate_10000op_10p(t *testing.T) { testSimulate(t, 10000, 10) } - -func TestSimulate_100op_100p(t *testing.T) { testSimulate(t, 100, 100) } -func TestSimulate_1000op_100p(t *testing.T) { testSimulate(t, 1000, 100) } -func TestSimulate_10000op_100p(t *testing.T) { testSimulate(t, 10000, 100) } - -func TestSimulate_10000op_1000p(t *testing.T) { testSimulate(t, 10000, 1000) } - -// Randomly generate operations on a given database with multiple clients to ensure consistency and thread safety. -func testSimulate(t *testing.T, threadCount, parallelism int) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - - rand.Seed(int64(qseed)) - - // A list of operations that readers and writers can perform. - var readerHandlers = []simulateHandler{simulateGetHandler} - var writerHandlers = []simulateHandler{simulateGetHandler, simulatePutHandler} - - var versions = make(map[int]*QuickDB) - versions[1] = NewQuickDB() - - db := MustOpenDB() - defer db.MustClose() - - var mutex sync.Mutex - - // Run n threads in parallel, each with their own operation. - var wg sync.WaitGroup - var threads = make(chan bool, parallelism) - var i int - for { - threads <- true - wg.Add(1) - writable := ((rand.Int() % 100) < 20) // 20% writers - - // Choose an operation to execute. - var handler simulateHandler - if writable { - handler = writerHandlers[rand.Intn(len(writerHandlers))] - } else { - handler = readerHandlers[rand.Intn(len(readerHandlers))] - } - - // Execute a thread for the given operation. - go func(writable bool, handler simulateHandler) { - defer wg.Done() - - // Start transaction. - tx, err := db.Begin(writable) - if err != nil { - t.Fatal("tx begin: ", err) - } - - // Obtain current state of the dataset. - mutex.Lock() - var qdb = versions[tx.ID()] - if writable { - qdb = versions[tx.ID()-1].Copy() - } - mutex.Unlock() - - // Make sure we commit/rollback the tx at the end and update the state. - if writable { - defer func() { - mutex.Lock() - versions[tx.ID()] = qdb - mutex.Unlock() - - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - }() - } else { - defer func() { _ = tx.Rollback() }() - } - - // Ignore operation if we don't have data yet. - if qdb == nil { - return - } - - // Execute handler. - handler(tx, qdb) - - // Release a thread back to the scheduling loop. - <-threads - }(writable, handler) - - i++ - if i > threadCount { - break - } - } - - // Wait until all threads are done. - wg.Wait() -} - -type simulateHandler func(tx *bolt.Tx, qdb *QuickDB) - -// Retrieves a key from the database and verifies that it is what is expected. -func simulateGetHandler(tx *bolt.Tx, qdb *QuickDB) { - // Randomly retrieve an existing exist. - keys := qdb.Rand() - if len(keys) == 0 { - return - } - - // Retrieve root bucket. - b := tx.Bucket(keys[0]) - if b == nil { - panic(fmt.Sprintf("bucket[0] expected: %08x\n", trunc(keys[0], 4))) - } - - // Drill into nested buckets. - for _, key := range keys[1 : len(keys)-1] { - b = b.Bucket(key) - if b == nil { - panic(fmt.Sprintf("bucket[n] expected: %v -> %v\n", keys, key)) - } - } - - // Verify key/value on the final bucket. - expected := qdb.Get(keys) - actual := b.Get(keys[len(keys)-1]) - if !bytes.Equal(actual, expected) { - fmt.Println("=== EXPECTED ===") - fmt.Println(expected) - fmt.Println("=== ACTUAL ===") - fmt.Println(actual) - fmt.Println("=== END ===") - panic("value mismatch") - } -} - -// Inserts a key into the database. -func simulatePutHandler(tx *bolt.Tx, qdb *QuickDB) { - var err error - keys, value := randKeys(), randValue() - - // Retrieve root bucket. - b := tx.Bucket(keys[0]) - if b == nil { - b, err = tx.CreateBucket(keys[0]) - if err != nil { - panic("create bucket: " + err.Error()) - } - } - - // Create nested buckets, if necessary. - for _, key := range keys[1 : len(keys)-1] { - child := b.Bucket(key) - if child != nil { - b = child - } else { - b, err = b.CreateBucket(key) - if err != nil { - panic("create bucket: " + err.Error()) - } - } - } - - // Insert into database. - if err := b.Put(keys[len(keys)-1], value); err != nil { - panic("put: " + err.Error()) - } - - // Insert into in-memory database. - qdb.Put(keys, value) -} - -// QuickDB is an in-memory database that replicates the functionality of the -// Bolt DB type except that it is entirely in-memory. It is meant for testing -// that the Bolt database is consistent. -type QuickDB struct { - sync.RWMutex - m map[string]interface{} -} - -// NewQuickDB returns an instance of QuickDB. -func NewQuickDB() *QuickDB { - return &QuickDB{m: make(map[string]interface{})} -} - -// Get retrieves the value at a key path. -func (db *QuickDB) Get(keys [][]byte) []byte { - db.RLock() - defer db.RUnlock() - - m := db.m - for _, key := range keys[:len(keys)-1] { - value := m[string(key)] - if value == nil { - return nil - } - switch value := value.(type) { - case map[string]interface{}: - m = value - case []byte: - return nil - } - } - - // Only return if it's a simple value. - if value, ok := m[string(keys[len(keys)-1])].([]byte); ok { - return value - } - return nil -} - -// Put inserts a value into a key path. -func (db *QuickDB) Put(keys [][]byte, value []byte) { - db.Lock() - defer db.Unlock() - - // Build buckets all the way down the key path. - m := db.m - for _, key := range keys[:len(keys)-1] { - if _, ok := m[string(key)].([]byte); ok { - return // Keypath intersects with a simple value. Do nothing. - } - - if m[string(key)] == nil { - m[string(key)] = make(map[string]interface{}) - } - m = m[string(key)].(map[string]interface{}) - } - - // Insert value into the last key. - m[string(keys[len(keys)-1])] = value -} - -// Rand returns a random key path that points to a simple value. -func (db *QuickDB) Rand() [][]byte { - db.RLock() - defer db.RUnlock() - if len(db.m) == 0 { - return nil - } - var keys [][]byte - db.rand(db.m, &keys) - return keys -} - -func (db *QuickDB) rand(m map[string]interface{}, keys *[][]byte) { - i, index := 0, rand.Intn(len(m)) - for k, v := range m { - if i == index { - *keys = append(*keys, []byte(k)) - if v, ok := v.(map[string]interface{}); ok { - db.rand(v, keys) - } - return - } - i++ - } - panic("quickdb rand: out-of-range") -} - -// Copy copies the entire database. -func (db *QuickDB) Copy() *QuickDB { - db.RLock() - defer db.RUnlock() - return &QuickDB{m: db.copy(db.m)} -} - -func (db *QuickDB) copy(m map[string]interface{}) map[string]interface{} { - clone := make(map[string]interface{}, len(m)) - for k, v := range m { - switch v := v.(type) { - case map[string]interface{}: - clone[k] = db.copy(v) - default: - clone[k] = v - } - } - return clone -} - -func randKey() []byte { - var min, max = 1, 1024 - n := rand.Intn(max-min) + min - b := make([]byte, n) - for i := 0; i < n; i++ { - b[i] = byte(rand.Intn(255)) - } - return b -} - -func randKeys() [][]byte { - var keys [][]byte - var count = rand.Intn(2) + 2 - for i := 0; i < count; i++ { - keys = append(keys, randKey()) - } - return keys -} - -func randValue() []byte { - n := rand.Intn(8192) - b := make([]byte, n) - for i := 0; i < n; i++ { - b[i] = byte(rand.Intn(255)) - } - return b -} diff --git a/vendor/github.com/boltdb/bolt/tx.go b/vendor/github.com/boltdb/bolt/tx.go index e74d2cae..6700308a 100644 --- a/vendor/github.com/boltdb/bolt/tx.go +++ b/vendor/github.com/boltdb/bolt/tx.go @@ -5,6 +5,7 @@ import ( "io" "os" "sort" + "strings" "time" "unsafe" ) @@ -202,8 +203,17 @@ func (tx *Tx) Commit() error { // If strict mode is enabled then perform a consistency check. // Only the first consistency error is reported in the panic. if tx.db.StrictMode { - if err, ok := <-tx.Check(); ok { - panic("check fail: " + err.Error()) + ch := tx.Check() + var errs []string + for { + err, ok := <-ch + if !ok { + break + } + errs = append(errs, err.Error()) + } + if len(errs) > 0 { + panic("check fail: " + strings.Join(errs, "\n")) } } @@ -297,12 +307,34 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { } defer func() { _ = f.Close() }() - // Copy the meta pages. - tx.db.metalock.Lock() - n, err = io.CopyN(w, f, int64(tx.db.pageSize*2)) - tx.db.metalock.Unlock() + // Generate a meta page. We use the same page data for both meta pages. + buf := make([]byte, tx.db.pageSize) + page := (*page)(unsafe.Pointer(&buf[0])) + page.flags = metaPageFlag + *page.meta() = *tx.meta + + // Write meta 0. + page.id = 0 + page.meta().checksum = page.meta().sum64() + nn, err := w.Write(buf) + n += int64(nn) if err != nil { - return n, fmt.Errorf("meta copy: %s", err) + return n, fmt.Errorf("meta 0 copy: %s", err) + } + + // Write meta 1 with a lower transaction id. + page.id = 1 + page.meta().txid -= 1 + page.meta().checksum = page.meta().sum64() + nn, err = w.Write(buf) + n += int64(nn) + if err != nil { + return n, fmt.Errorf("meta 1 copy: %s", err) + } + + // Move past the meta pages in the file. + if _, err := f.Seek(int64(tx.db.pageSize*2), os.SEEK_SET); err != nil { + return n, fmt.Errorf("seek: %s", err) } // Copy data pages. @@ -349,7 +381,9 @@ func (tx *Tx) Check() <-chan error { func (tx *Tx) check(ch chan error) { // Check if any pages are double freed. freed := make(map[pgid]bool) - for _, id := range tx.db.freelist.all() { + all := make([]pgid, tx.db.freelist.count()) + tx.db.freelist.copyall(all) + for _, id := range all { if freed[id] { ch <- fmt.Errorf("page %d: already freed", id) } @@ -441,6 +475,8 @@ func (tx *Tx) write() error { for _, p := range tx.pages { pages = append(pages, p) } + // Clear out page cache early. + tx.pages = make(map[pgid]*page) sort.Sort(pages) // Write pages to disk in order. @@ -485,8 +521,22 @@ func (tx *Tx) write() error { } } - // Clear out page cache. - tx.pages = make(map[pgid]*page) + // Put small pages back to page pool. + for _, p := range pages { + // Ignore page sizes over 1 page. + // These are allocated using make() instead of the page pool. + if int(p.overflow) != 0 { + continue + } + + buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:tx.db.pageSize] + + // See https://go.googlesource.com/go/+/f03c9202c43e0abb130669852082117ca50aa9b1 + for i := range buf { + buf[i] = 0 + } + tx.db.pagePool.Put(buf) + } return nil } diff --git a/vendor/github.com/boltdb/bolt/tx_test.go b/vendor/github.com/boltdb/bolt/tx_test.go deleted file mode 100644 index 18ff1669..00000000 --- a/vendor/github.com/boltdb/bolt/tx_test.go +++ /dev/null @@ -1,716 +0,0 @@ -package bolt_test - -import ( - "bytes" - "errors" - "fmt" - "log" - "os" - "testing" - - "github.com/boltdb/bolt" -) - -// Ensure that committing a closed transaction returns an error. -func TestTx_Commit_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - - if _, err := tx.CreateBucket([]byte("foo")); err != nil { - t.Fatal(err) - } - - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - - if err := tx.Commit(); err != bolt.ErrTxClosed { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that rolling back a closed transaction returns an error. -func TestTx_Rollback_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - - if err := tx.Rollback(); err != nil { - t.Fatal(err) - } - if err := tx.Rollback(); err != bolt.ErrTxClosed { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that committing a read-only transaction returns an error. -func TestTx_Commit_ErrTxNotWritable(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - tx, err := db.Begin(false) - if err != nil { - t.Fatal(err) - } - if err := tx.Commit(); err != bolt.ErrTxNotWritable { - t.Fatal(err) - } -} - -// Ensure that a transaction can retrieve a cursor on the root bucket. -func TestTx_Cursor(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - - if _, err := tx.CreateBucket([]byte("woojits")); err != nil { - t.Fatal(err) - } - - c := tx.Cursor() - if k, v := c.First(); !bytes.Equal(k, []byte("widgets")) { - t.Fatalf("unexpected key: %v", k) - } else if v != nil { - t.Fatalf("unexpected value: %v", v) - } - - if k, v := c.Next(); !bytes.Equal(k, []byte("woojits")) { - t.Fatalf("unexpected key: %v", k) - } else if v != nil { - t.Fatalf("unexpected value: %v", v) - } - - if k, v := c.Next(); k != nil { - t.Fatalf("unexpected key: %v", k) - } else if v != nil { - t.Fatalf("unexpected value: %v", k) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that creating a bucket with a read-only transaction returns an error. -func TestTx_CreateBucket_ErrTxNotWritable(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.View(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("foo")) - if err != bolt.ErrTxNotWritable { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that creating a bucket on a closed transaction returns an error. -func TestTx_CreateBucket_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - - if _, err := tx.CreateBucket([]byte("foo")); err != bolt.ErrTxClosed { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that a Tx can retrieve a bucket. -func TestTx_Bucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - if tx.Bucket([]byte("widgets")) == nil { - t.Fatal("expected bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a Tx retrieving a non-existent key returns nil. -func TestTx_Get_NotFound(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if b.Get([]byte("no_such_key")) != nil { - t.Fatal("expected nil value") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can be created and retrieved. -func TestTx_CreateBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - // Create a bucket. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } else if b == nil { - t.Fatal("expected bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Read the bucket through a separate transaction. - if err := db.View(func(tx *bolt.Tx) error { - if tx.Bucket([]byte("widgets")) == nil { - t.Fatal("expected bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can be created if it doesn't already exist. -func TestTx_CreateBucketIfNotExists(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - // Create bucket. - if b, err := tx.CreateBucketIfNotExists([]byte("widgets")); err != nil { - t.Fatal(err) - } else if b == nil { - t.Fatal("expected bucket") - } - - // Create bucket again. - if b, err := tx.CreateBucketIfNotExists([]byte("widgets")); err != nil { - t.Fatal(err) - } else if b == nil { - t.Fatal("expected bucket") - } - - return nil - }); err != nil { - t.Fatal(err) - } - - // Read the bucket through a separate transaction. - if err := db.View(func(tx *bolt.Tx) error { - if tx.Bucket([]byte("widgets")) == nil { - t.Fatal("expected bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure transaction returns an error if creating an unnamed bucket. -func TestTx_CreateBucketIfNotExists_ErrBucketNameRequired(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucketIfNotExists([]byte{}); err != bolt.ErrBucketNameRequired { - t.Fatalf("unexpected error: %s", err) - } - - if _, err := tx.CreateBucketIfNotExists(nil); err != bolt.ErrBucketNameRequired { - t.Fatalf("unexpected error: %s", err) - } - - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket cannot be created twice. -func TestTx_CreateBucket_ErrBucketExists(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - // Create a bucket. - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Create the same bucket again. - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket([]byte("widgets")); err != bolt.ErrBucketExists { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket is created with a non-blank name. -func TestTx_CreateBucket_ErrBucketNameRequired(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucket(nil); err != bolt.ErrBucketNameRequired { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that a bucket can be deleted. -func TestTx_DeleteBucket(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - // Create a bucket and add a value. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - // Delete the bucket and make sure we can't get the value. - if err := db.Update(func(tx *bolt.Tx) error { - if err := tx.DeleteBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - if tx.Bucket([]byte("widgets")) != nil { - t.Fatal("unexpected bucket") - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - // Create the bucket again and make sure there's not a phantom value. - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if v := b.Get([]byte("foo")); v != nil { - t.Fatalf("unexpected phantom value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that deleting a bucket on a closed transaction returns an error. -func TestTx_DeleteBucket_ErrTxClosed(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - tx, err := db.Begin(true) - if err != nil { - t.Fatal(err) - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - if err := tx.DeleteBucket([]byte("foo")); err != bolt.ErrTxClosed { - t.Fatalf("unexpected error: %s", err) - } -} - -// Ensure that deleting a bucket with a read-only transaction returns an error. -func TestTx_DeleteBucket_ReadOnly(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.View(func(tx *bolt.Tx) error { - if err := tx.DeleteBucket([]byte("foo")); err != bolt.ErrTxNotWritable { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that nothing happens when deleting a bucket that doesn't exist. -func TestTx_DeleteBucket_NotFound(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - if err := tx.DeleteBucket([]byte("widgets")); err != bolt.ErrBucketNotFound { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that no error is returned when a tx.ForEach function does not return -// an error. -func TestTx_ForEach_NoError(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - - if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { - return nil - }); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that an error is returned when a tx.ForEach function returns an error. -func TestTx_ForEach_WithError(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - - marker := errors.New("marker") - if err := tx.ForEach(func(name []byte, b *bolt.Bucket) error { - return marker - }); err != marker { - t.Fatalf("unexpected error: %s", err) - } - return nil - }); err != nil { - t.Fatal(err) - } -} - -// Ensure that Tx commit handlers are called after a transaction successfully commits. -func TestTx_OnCommit(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var x int - if err := db.Update(func(tx *bolt.Tx) error { - tx.OnCommit(func() { x += 1 }) - tx.OnCommit(func() { x += 2 }) - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } else if x != 3 { - t.Fatalf("unexpected x: %d", x) - } -} - -// Ensure that Tx commit handlers are NOT called after a transaction rolls back. -func TestTx_OnCommit_Rollback(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - var x int - if err := db.Update(func(tx *bolt.Tx) error { - tx.OnCommit(func() { x += 1 }) - tx.OnCommit(func() { x += 2 }) - if _, err := tx.CreateBucket([]byte("widgets")); err != nil { - t.Fatal(err) - } - return errors.New("rollback this commit") - }); err == nil || err.Error() != "rollback this commit" { - t.Fatalf("unexpected error: %s", err) - } else if x != 0 { - t.Fatalf("unexpected x: %d", x) - } -} - -// Ensure that the database can be copied to a file path. -func TestTx_CopyFile(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - - path := tempfile() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte("bat")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - return tx.CopyFile(path, 0600) - }); err != nil { - t.Fatal(err) - } - - db2, err := bolt.Open(path, 0600, nil) - if err != nil { - t.Fatal(err) - } - - if err := db2.View(func(tx *bolt.Tx) error { - if v := tx.Bucket([]byte("widgets")).Get([]byte("foo")); !bytes.Equal(v, []byte("bar")) { - t.Fatalf("unexpected value: %v", v) - } - if v := tx.Bucket([]byte("widgets")).Get([]byte("baz")); !bytes.Equal(v, []byte("bat")) { - t.Fatalf("unexpected value: %v", v) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db2.Close(); err != nil { - t.Fatal(err) - } -} - -type failWriterError struct{} - -func (failWriterError) Error() string { - return "error injected for tests" -} - -type failWriter struct { - // fail after this many bytes - After int -} - -func (f *failWriter) Write(p []byte) (n int, err error) { - n = len(p) - if n > f.After { - n = f.After - err = failWriterError{} - } - f.After -= n - return n, err -} - -// Ensure that Copy handles write errors right. -func TestTx_CopyFile_Error_Meta(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte("bat")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - return tx.Copy(&failWriter{}) - }); err == nil || err.Error() != "meta copy: error injected for tests" { - t.Fatalf("unexpected error: %v", err) - } -} - -// Ensure that Copy handles write errors right. -func TestTx_CopyFile_Error_Normal(t *testing.T) { - db := MustOpenDB() - defer db.MustClose() - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - t.Fatal(err) - } - if err := b.Put([]byte("baz"), []byte("bat")); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - - if err := db.View(func(tx *bolt.Tx) error { - return tx.Copy(&failWriter{3 * db.Info().PageSize}) - }); err == nil || err.Error() != "error injected for tests" { - t.Fatalf("unexpected error: %v", err) - } -} - -func ExampleTx_Rollback() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Create a bucket. - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucket([]byte("widgets")) - return err - }); err != nil { - log.Fatal(err) - } - - // Set a value for a key. - if err := db.Update(func(tx *bolt.Tx) error { - return tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("bar")) - }); err != nil { - log.Fatal(err) - } - - // Update the key but rollback the transaction so it never saves. - tx, err := db.Begin(true) - if err != nil { - log.Fatal(err) - } - b := tx.Bucket([]byte("widgets")) - if err := b.Put([]byte("foo"), []byte("baz")); err != nil { - log.Fatal(err) - } - if err := tx.Rollback(); err != nil { - log.Fatal(err) - } - - // Ensure that our original value is still set. - if err := db.View(func(tx *bolt.Tx) error { - value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value for 'foo' is still: %s\n", value) - return nil - }); err != nil { - log.Fatal(err) - } - - // Close database to release file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // The value for 'foo' is still: bar -} - -func ExampleTx_CopyFile() { - // Open the database. - db, err := bolt.Open(tempfile(), 0666, nil) - if err != nil { - log.Fatal(err) - } - defer os.Remove(db.Path()) - - // Create a bucket and a key. - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket([]byte("widgets")) - if err != nil { - return err - } - if err := b.Put([]byte("foo"), []byte("bar")); err != nil { - return err - } - return nil - }); err != nil { - log.Fatal(err) - } - - // Copy the database to another file. - toFile := tempfile() - if err := db.View(func(tx *bolt.Tx) error { - return tx.CopyFile(toFile, 0666) - }); err != nil { - log.Fatal(err) - } - defer os.Remove(toFile) - - // Open the cloned database. - db2, err := bolt.Open(toFile, 0666, nil) - if err != nil { - log.Fatal(err) - } - - // Ensure that the key exists in the copy. - if err := db2.View(func(tx *bolt.Tx) error { - value := tx.Bucket([]byte("widgets")).Get([]byte("foo")) - fmt.Printf("The value for 'foo' in the clone is: %s\n", value) - return nil - }); err != nil { - log.Fatal(err) - } - - // Close database to release file lock. - if err := db.Close(); err != nil { - log.Fatal(err) - } - - if err := db2.Close(); err != nil { - log.Fatal(err) - } - - // Output: - // The value for 'foo' in the clone is: bar -} diff --git a/vendor/github.com/cpuguy83/go-md2man/md2man/md2man.go b/vendor/github.com/cpuguy83/go-md2man/md2man/md2man.go deleted file mode 100644 index 8f44fa15..00000000 --- a/vendor/github.com/cpuguy83/go-md2man/md2man/md2man.go +++ /dev/null @@ -1,19 +0,0 @@ -package md2man - -import ( - "github.com/russross/blackfriday" -) - -func Render(doc []byte) []byte { - renderer := RoffRenderer(0) - extensions := 0 - extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS - extensions |= blackfriday.EXTENSION_TABLES - extensions |= blackfriday.EXTENSION_FENCED_CODE - extensions |= blackfriday.EXTENSION_AUTOLINK - extensions |= blackfriday.EXTENSION_SPACE_HEADERS - extensions |= blackfriday.EXTENSION_FOOTNOTES - extensions |= blackfriday.EXTENSION_TITLEBLOCK - - return blackfriday.Markdown(doc, renderer, extensions) -} diff --git a/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go b/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go deleted file mode 100644 index 9f12daa1..00000000 --- a/vendor/github.com/cpuguy83/go-md2man/md2man/roff.go +++ /dev/null @@ -1,281 +0,0 @@ -package md2man - -import ( - "bytes" - "fmt" - "html" - "strings" - - "github.com/russross/blackfriday" -) - -type roffRenderer struct{} - -var listCounter int - -func RoffRenderer(flags int) blackfriday.Renderer { - return &roffRenderer{} -} - -func (r *roffRenderer) GetFlags() int { - return 0 -} - -func (r *roffRenderer) TitleBlock(out *bytes.Buffer, text []byte) { - out.WriteString(".TH ") - - splitText := bytes.Split(text, []byte("\n")) - for i, line := range splitText { - line = bytes.TrimPrefix(line, []byte("% ")) - if i == 0 { - line = bytes.Replace(line, []byte("("), []byte("\" \""), 1) - line = bytes.Replace(line, []byte(")"), []byte("\" \""), 1) - } - line = append([]byte("\""), line...) - line = append(line, []byte("\" ")...) - out.Write(line) - } - out.WriteString("\n") - - // disable hyphenation - out.WriteString(".nh\n") - // disable justification (adjust text to left margin only) - out.WriteString(".ad l\n") -} - -func (r *roffRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) { - out.WriteString("\n.PP\n.RS\n\n.nf\n") - escapeSpecialChars(out, text) - out.WriteString("\n.fi\n.RE\n") -} - -func (r *roffRenderer) BlockQuote(out *bytes.Buffer, text []byte) { - out.WriteString("\n.PP\n.RS\n") - out.Write(text) - out.WriteString("\n.RE\n") -} - -func (r *roffRenderer) BlockHtml(out *bytes.Buffer, text []byte) { - out.Write(text) -} - -func (r *roffRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) { - marker := out.Len() - - switch { - case marker == 0: - // This is the doc header - out.WriteString(".TH ") - case level == 1: - out.WriteString("\n\n.SH ") - case level == 2: - out.WriteString("\n.SH ") - default: - out.WriteString("\n.SS ") - } - - if !text() { - out.Truncate(marker) - return - } -} - -func (r *roffRenderer) HRule(out *bytes.Buffer) { - out.WriteString("\n.ti 0\n\\l'\\n(.lu'\n") -} - -func (r *roffRenderer) List(out *bytes.Buffer, text func() bool, flags int) { - marker := out.Len() - if flags&blackfriday.LIST_TYPE_ORDERED != 0 { - listCounter = 1 - } - if !text() { - out.Truncate(marker) - return - } -} - -func (r *roffRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) { - if flags&blackfriday.LIST_TYPE_ORDERED != 0 { - out.WriteString(fmt.Sprintf(".IP \"%3d.\" 5\n", listCounter)) - listCounter += 1 - } else { - out.WriteString(".IP \\(bu 2\n") - } - out.Write(text) - out.WriteString("\n") -} - -func (r *roffRenderer) Paragraph(out *bytes.Buffer, text func() bool) { - marker := out.Len() - out.WriteString("\n.PP\n") - if !text() { - out.Truncate(marker) - return - } - if marker != 0 { - out.WriteString("\n") - } -} - -// TODO: This might now work -func (r *roffRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) { - out.WriteString(".TS\nallbox;\n") - - out.Write(header) - out.Write(body) - out.WriteString("\n.TE\n") -} - -func (r *roffRenderer) TableRow(out *bytes.Buffer, text []byte) { - if out.Len() > 0 { - out.WriteString("\n") - } - out.Write(text) - out.WriteString("\n") -} - -func (r *roffRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, align int) { - if out.Len() > 0 { - out.WriteString(" ") - } - out.Write(text) - out.WriteString(" ") -} - -// TODO: This is probably broken -func (r *roffRenderer) TableCell(out *bytes.Buffer, text []byte, align int) { - if out.Len() > 0 { - out.WriteString("\t") - } - out.Write(text) - out.WriteString("\t") -} - -func (r *roffRenderer) Footnotes(out *bytes.Buffer, text func() bool) { - -} - -func (r *roffRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) { - -} - -func (r *roffRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) { - out.WriteString("\n\\[la]") - out.Write(link) - out.WriteString("\\[ra]") -} - -func (r *roffRenderer) CodeSpan(out *bytes.Buffer, text []byte) { - out.WriteString("\\fB\\fC") - escapeSpecialChars(out, text) - out.WriteString("\\fR") -} - -func (r *roffRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) { - out.WriteString("\\fB") - out.Write(text) - out.WriteString("\\fP") -} - -func (r *roffRenderer) Emphasis(out *bytes.Buffer, text []byte) { - out.WriteString("\\fI") - out.Write(text) - out.WriteString("\\fP") -} - -func (r *roffRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) { -} - -func (r *roffRenderer) LineBreak(out *bytes.Buffer) { - out.WriteString("\n.br\n") -} - -func (r *roffRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) { - r.AutoLink(out, link, 0) -} - -func (r *roffRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) { - out.Write(tag) -} - -func (r *roffRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) { - out.WriteString("\\s+2") - out.Write(text) - out.WriteString("\\s-2") -} - -func (r *roffRenderer) StrikeThrough(out *bytes.Buffer, text []byte) { -} - -func (r *roffRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) { - -} - -func (r *roffRenderer) Entity(out *bytes.Buffer, entity []byte) { - out.WriteString(html.UnescapeString(string(entity))) -} - -func processFooterText(text []byte) []byte { - text = bytes.TrimPrefix(text, []byte("% ")) - newText := []byte{} - textArr := strings.Split(string(text), ") ") - - for i, w := range textArr { - if i == 0 { - w = strings.Replace(w, "(", "\" \"", 1) - w = fmt.Sprintf("\"%s\"", w) - } else { - w = fmt.Sprintf(" \"%s\"", w) - } - newText = append(newText, []byte(w)...) - } - newText = append(newText, []byte(" \"\"")...) - - return newText -} - -func (r *roffRenderer) NormalText(out *bytes.Buffer, text []byte) { - escapeSpecialChars(out, text) -} - -func (r *roffRenderer) DocumentHeader(out *bytes.Buffer) { -} - -func (r *roffRenderer) DocumentFooter(out *bytes.Buffer) { -} - -func needsBackslash(c byte) bool { - for _, r := range []byte("-_&\\~") { - if c == r { - return true - } - } - return false -} - -func escapeSpecialChars(out *bytes.Buffer, text []byte) { - for i := 0; i < len(text); i++ { - // escape initial apostrophe or period - if len(text) >= 1 && (text[0] == '\'' || text[0] == '.') { - out.WriteString("\\&") - } - - // directly copy normal characters - org := i - - for i < len(text) && !needsBackslash(text[i]) { - i++ - } - if i > org { - out.Write(text[org:i]) - } - - // escape a character - if i >= len(text) { - break - } - out.WriteByte('\\') - out.WriteByte(text[i]) - } -} diff --git a/vendor/github.com/cznic/b/AUTHORS b/vendor/github.com/cznic/b/AUTHORS deleted file mode 100644 index 0078f5f5..00000000 --- a/vendor/github.com/cznic/b/AUTHORS +++ /dev/null @@ -1,11 +0,0 @@ -# This file lists authors for copyright purposes. This file is distinct from -# the CONTRIBUTORS files. See the latter for an explanation. -# -# Names should be added to this file as: -# Name or Organization -# -# The email address is not required for organizations. -# -# Please keep the list sorted. - -Jan Mercl <0xjnml@gmail.com> diff --git a/vendor/github.com/cznic/b/CONTRIBUTORS b/vendor/github.com/cznic/b/CONTRIBUTORS deleted file mode 100644 index 6541dbc4..00000000 --- a/vendor/github.com/cznic/b/CONTRIBUTORS +++ /dev/null @@ -1,12 +0,0 @@ -# This file lists people who contributed code to this repository. The AUTHORS -# file lists the copyright holders; this file lists people. -# -# Names should be added to this file like so: -# Name -# -# Please keep the list sorted. - -Brian Fallik -Dan Kortschak -Jan Mercl <0xjnml@gmail.com> -Nikifor Seryakov diff --git a/vendor/github.com/cznic/b/Makefile b/vendor/github.com/cznic/b/Makefile deleted file mode 100644 index b02c86db..00000000 --- a/vendor/github.com/cznic/b/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2014 The b Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -.PHONY: all todo clean cover generic mem nuke cpu - -testbin=b.test - -all: editor - go build - go vet - golint . - go install - make todo - -editor: - gofmt -l -s -w . - go test -i - go test - -clean: - @go clean - rm -f *~ *.out $(testbin) - -cover: - t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t - -cpu: - go test -c - ./$(testbin) -test.cpuprofile cpu.out - go tool pprof --lines $(testbin) cpu.out - -generic: - @# writes to stdout a version where the type of key is KEY and the type - @# of value is VALUE. - @# - @# Intended use is to replace all textual occurrences of KEY or VALUE in - @# the output with your desired types. - @sed -e 's|interface{}[^{]*/\*K\*/|KEY|g' -e 's|interface{}[^{]*/\*V\*/|VALUE|g' btree.go - -mem: - go test -c - ./$(testbin) -test.bench . -test.memprofile mem.out -test.memprofilerate 1 - go tool pprof --lines --web --alloc_space $(testbin) mem.out - -nuke: clean - rm -f *.test *.out - -todo: - @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go || true - @grep -n TODO *.go || true - @grep -n BUG *.go || true - @grep -n println *.go || true diff --git a/vendor/github.com/cznic/b/README.md b/vendor/github.com/cznic/b/README.md deleted file mode 100644 index 5a2d8030..00000000 --- a/vendor/github.com/cznic/b/README.md +++ /dev/null @@ -1,10 +0,0 @@ -b -= - -Package b implements a B+tree. - -Installation: - - $ go get github.com/cznic/b - -Documentation: [godoc.org/github.com/cznic/b](http://godoc.org/github.com/cznic/b) diff --git a/vendor/github.com/cznic/b/all_test.go b/vendor/github.com/cznic/b/all_test.go deleted file mode 100644 index c1b9b444..00000000 --- a/vendor/github.com/cznic/b/all_test.go +++ /dev/null @@ -1,1349 +0,0 @@ -// Copyright 2014 The b Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package b - -import ( - "bytes" - "fmt" - "io" - "math" - "path" - "runtime" - "runtime/debug" - "strings" - "testing" - - "github.com/cznic/mathutil" - "github.com/cznic/strutil" -) - -var caller = func(s string, va ...interface{}) { - _, fn, fl, _ := runtime.Caller(2) - fmt.Printf("%s:%d: ", path.Base(fn), fl) - fmt.Printf(s, va...) - fmt.Println() -} - -func dbg(s string, va ...interface{}) { - if s == "" { - s = strings.Repeat("%v ", len(va)) - } - _, fn, fl, _ := runtime.Caller(1) - fmt.Printf("%s:%d: ", path.Base(fn), fl) - fmt.Printf(s, va...) - fmt.Println() -} - -func TODO(...interface{}) string { - _, fn, fl, _ := runtime.Caller(1) - return fmt.Sprintf("TODO: %s:%d:\n", path.Base(fn), fl) -} - -func use(...interface{}) {} - -// ============================================================================ - -func isNil(p interface{}) bool { - switch x := p.(type) { - case *x: - if x == nil { - return true - } - case *d: - if x == nil { - return true - } - } - return false -} - -func (t *Tree) dump() string { - var buf bytes.Buffer - f := strutil.IndentFormatter(&buf, "\t") - - num := map[interface{}]int{} - visited := map[interface{}]bool{} - - handle := func(p interface{}) int { - if isNil(p) { - return 0 - } - - if n, ok := num[p]; ok { - return n - } - - n := len(num) + 1 - num[p] = n - return n - } - - var pagedump func(interface{}, string) - pagedump = func(p interface{}, pref string) { - if isNil(p) || visited[p] { - return - } - - visited[p] = true - switch x := p.(type) { - case *x: - h := handle(p) - n := 0 - for i, v := range x.x { - if v.ch != nil || v.k != nil { - n = i + 1 - } - } - f.Format("%sX#%d(%p) n %d:%d {", pref, h, x, x.c, n) - a := []interface{}{} - for i, v := range x.x[:n] { - a = append(a, v.ch) - if i != 0 { - f.Format(" ") - } - f.Format("(C#%d K %v)", handle(v.ch), v.k) - } - f.Format("}\n") - for _, p := range a { - pagedump(p, pref+". ") - } - case *d: - h := handle(p) - n := 0 - for i, v := range x.d { - if v.k != nil || v.v != nil { - n = i + 1 - } - } - f.Format("%sD#%d(%p) P#%d N#%d n %d:%d {", pref, h, x, handle(x.p), handle(x.n), x.c, n) - for i, v := range x.d[:n] { - if i != 0 { - f.Format(" ") - } - f.Format("%v:%v", v.k, v.v) - } - f.Format("}\n") - } - } - - pagedump(t.r, "") - s := buf.String() - if s != "" { - s = s[:len(s)-1] - } - return s -} - -func rng() *mathutil.FC32 { - x, err := mathutil.NewFC32(math.MinInt32/4, math.MaxInt32/4, false) - if err != nil { - panic(err) - } - - return x -} - -func cmp(a, b interface{}) int { - return a.(int) - b.(int) -} - -func TestGet0(t *testing.T) { - r := TreeNew(cmp) - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - _, ok := r.Get(42) - if ok { - t.Fatal(ok) - } - -} - -func TestSetGet0(t *testing.T) { - r := TreeNew(cmp) - set := r.Set - set(42, 314) - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - v, ok := r.Get(42) - if !ok { - t.Fatal(ok) - } - - if g, e := v.(int), 314; g != e { - t.Fatal(g, e) - } - - set(42, 278) - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - v, ok = r.Get(42) - if !ok { - t.Fatal(ok) - } - - if g, e := v.(int), 278; g != e { - t.Fatal(g, e) - } - - set(420, 0.5) - if g, e := r.Len(), 2; g != e { - t.Fatal(g, e) - } - - v, ok = r.Get(42) - if !ok { - t.Fatal(ok) - } - - if g, e := v.(int), 278; g != e { - t.Fatal(g, e) - } - - v, ok = r.Get(420) - if !ok { - t.Fatal(ok) - } - - if g, e := v.(float64), 0.5; g != e { - t.Fatal(g, e) - } -} - -func TestSetGet1(t *testing.T) { - const N = 40000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - for i := range a { - a[i] = (i ^ x) << 1 - } - for i, k := range a { - set(k, k^x) - if g, e := r.Len(), i+1; g != e { - t.Fatal(i, g, e) - } - } - - for i, k := range a { - v, ok := r.Get(k) - if !ok { - t.Fatal(i, k, v, ok) - } - - if g, e := v.(int), k^x; g != e { - t.Fatal(i, g, e) - } - - k |= 1 - _, ok = r.Get(k) - if ok { - t.Fatal(i, k) - } - - } - - for _, k := range a { - r.Set(k, (k^x)+42) - } - - for i, k := range a { - v, ok := r.Get(k) - if !ok { - t.Fatal(i, k, v, ok) - } - - if g, e := v.(int), k^x+42; g != e { - t.Fatal(i, g, e) - } - - k |= 1 - _, ok = r.Get(k) - if ok { - t.Fatal(i, k) - } - } - } -} - -func TestPrealloc(*testing.T) { - const n = 2e6 - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - r := TreeNew(cmp) - for _, v := range a { - r.Set(v, 0) - } - r.Close() -} - -func BenchmarkSetSeq1e3(b *testing.B) { - benchmarkSetSeq(b, 1e3) -} - -func BenchmarkSetSeq1e4(b *testing.B) { - benchmarkSetSeq(b, 1e4) -} - -func BenchmarkSetSeq1e5(b *testing.B) { - benchmarkSetSeq(b, 1e5) -} - -func BenchmarkSetSeq1e6(b *testing.B) { - benchmarkSetSeq(b, 1e6) -} - -func benchmarkSetSeq(b *testing.B, n int) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - debug.FreeOSMemory() - b.StartTimer() - for j := 0; j < n; j++ { - r.Set(j, j) - } - b.StopTimer() - r.Close() - } - b.StopTimer() -} - -func BenchmarkGetSeq1e3(b *testing.B) { - benchmarkGetSeq(b, 1e3) -} - -func BenchmarkGetSeq1e4(b *testing.B) { - benchmarkGetSeq(b, 1e4) -} - -func BenchmarkGetSeq1e5(b *testing.B) { - benchmarkGetSeq(b, 1e5) -} - -func BenchmarkGetSeq1e6(b *testing.B) { - benchmarkGetSeq(b, 1e6) -} - -func benchmarkGetSeq(b *testing.B, n int) { - r := TreeNew(cmp) - for i := 0; i < n; i++ { - r.Set(i, i) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < n; j++ { - r.Get(j) - } - } - b.StopTimer() - r.Close() -} - -func BenchmarkSetRnd1e3(b *testing.B) { - benchmarkSetRnd(b, 1e3) -} - -func BenchmarkSetRnd1e4(b *testing.B) { - benchmarkSetRnd(b, 1e4) -} - -func BenchmarkSetRnd1e5(b *testing.B) { - benchmarkSetRnd(b, 1e5) -} - -func BenchmarkSetRnd1e6(b *testing.B) { - benchmarkSetRnd(b, 1e6) -} - -func benchmarkSetRnd(b *testing.B, n int) { - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - debug.FreeOSMemory() - b.StartTimer() - for _, v := range a { - r.Set(v, 0) - } - b.StopTimer() - r.Close() - } - b.StopTimer() -} - -func BenchmarkGetRnd1e3(b *testing.B) { - benchmarkGetRnd(b, 1e3) -} - -func BenchmarkGetRnd1e4(b *testing.B) { - benchmarkGetRnd(b, 1e4) -} - -func BenchmarkGetRnd1e5(b *testing.B) { - benchmarkGetRnd(b, 1e5) -} - -func BenchmarkGetRnd1e6(b *testing.B) { - benchmarkGetRnd(b, 1e6) -} - -func benchmarkGetRnd(b *testing.B, n int) { - r := TreeNew(cmp) - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - for _, v := range a { - r.Set(v, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, v := range a { - r.Get(v) - } - } - b.StopTimer() - r.Close() -} - -func TestSetGet2(t *testing.T) { - const N = 40000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { - rng := rng() - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - for i := range a { - a[i] = (rng.Next() ^ x) << 1 - } - for i, k := range a { - set(k, k^x) - if g, e := r.Len(), i+1; g != e { - t.Fatal(i, x, g, e) - } - } - - for i, k := range a { - v, ok := r.Get(k) - if !ok { - t.Fatal(i, k, v, ok) - } - - if g, e := v.(int), k^x; g != e { - t.Fatal(i, g, e) - } - - k |= 1 - _, ok = r.Get(k) - if ok { - t.Fatal(i, k) - } - } - - for _, k := range a { - r.Set(k, (k^x)+42) - } - - for i, k := range a { - v, ok := r.Get(k) - if !ok { - t.Fatal(i, k, v, ok) - } - - if g, e := v.(int), k^x+42; g != e { - t.Fatal(i, g, e) - } - - k |= 1 - _, ok = r.Get(k) - if ok { - t.Fatal(i, k) - } - } - } -} - -func TestSetGet3(t *testing.T) { - r := TreeNew(cmp) - set := r.Set - var i int - for i = 0; ; i++ { - set(i, -i) - if _, ok := r.r.(*x); ok { - break - } - } - for j := 0; j <= i; j++ { - set(j, j) - } - - for j := 0; j <= i; j++ { - v, ok := r.Get(j) - if !ok { - t.Fatal(j) - } - - if g, e := v.(int), j; g != e { - t.Fatal(g, e) - } - } -} - -func TestDelete0(t *testing.T) { - r := TreeNew(cmp) - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - r.Set(0, 0) - if ok := r.Delete(1); ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - r.Set(0, 0) - r.Set(1, 1) - if ok := r.Delete(1); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(1); ok { - t.Fatal(ok) - } - - if ok := r.Delete(0); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - r.Set(0, 0) - r.Set(1, 1) - if ok := r.Delete(0); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - if ok := r.Delete(1); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(1); ok { - t.Fatal(ok) - } -} - -func TestDelete1(t *testing.T) { - const N = 130000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - for i := range a { - a[i] = (i ^ x) << 1 - } - for _, k := range a { - set(k, 0) - } - - for i, k := range a { - ok := r.Delete(k) - if !ok { - t.Fatal(i, x, k) - } - - if g, e := r.Len(), N-i-1; g != e { - t.Fatal(i, g, e) - } - } - } -} - -func BenchmarkDelSeq1e3(b *testing.B) { - benchmarkDelSeq(b, 1e3) -} - -func BenchmarkDelSeq1e4(b *testing.B) { - benchmarkDelSeq(b, 1e4) -} - -func BenchmarkDelSeq1e5(b *testing.B) { - benchmarkDelSeq(b, 1e5) -} - -func BenchmarkDelSeq1e6(b *testing.B) { - benchmarkDelSeq(b, 1e6) -} - -func benchmarkDelSeq(b *testing.B, n int) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - for i := 0; i < n; i++ { - r.Set(i, i) - } - debug.FreeOSMemory() - b.StartTimer() - for j := 0; j < n; j++ { - r.Delete(j) - } - } - b.StopTimer() -} - -func BenchmarkDelRnd1e3(b *testing.B) { - benchmarkDelRnd(b, 1e3) -} - -func BenchmarkDelRnd1e4(b *testing.B) { - benchmarkDelRnd(b, 1e4) -} - -func BenchmarkDelRnd1e5(b *testing.B) { - benchmarkDelRnd(b, 1e5) -} - -func BenchmarkDelRnd1e6(b *testing.B) { - benchmarkDelRnd(b, 1e6) -} - -func benchmarkDelRnd(b *testing.B, n int) { - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - for _, v := range a { - r.Set(v, 0) - } - debug.FreeOSMemory() - b.StartTimer() - for _, v := range a { - r.Delete(v) - } - b.StopTimer() - r.Close() - } - b.StopTimer() -} - -func TestDelete2(t *testing.T) { - const N = 100000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} { - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - rng := rng() - for i := range a { - a[i] = (rng.Next() ^ x) << 1 - } - for _, k := range a { - set(k, 0) - } - - for i, k := range a { - ok := r.Delete(k) - if !ok { - t.Fatal(i, x, k) - } - - if g, e := r.Len(), N-i-1; g != e { - t.Fatal(i, g, e) - } - } - } -} - -func TestEnumeratorNext(t *testing.T) { - // seeking within 3 keys: 10, 20, 30 - table := []struct { - k int - hit bool - keys []int - }{ - {5, false, []int{10, 20, 30}}, - {10, true, []int{10, 20, 30}}, - {15, false, []int{20, 30}}, - {20, true, []int{20, 30}}, - {25, false, []int{30}}, - {30, true, []int{30}}, - {35, false, []int{}}, - } - - for i, test := range table { - up := test.keys - r := TreeNew(cmp) - - r.Set(10, 100) - r.Set(20, 200) - r.Set(30, 300) - - for verChange := 0; verChange < 16; verChange++ { - en, hit := r.Seek(test.k) - - if g, e := hit, test.hit; g != e { - t.Fatal(i, g, e) - } - - j := 0 - for { - if verChange&(1<= len(up) { - t.Fatal(i, j, verChange) - } - - if g, e := k.(int), up[j]; g != e { - t.Fatal(i, j, verChange, g, e) - } - - if g, e := v.(int), 10*up[j]; g != e { - t.Fatal(i, g, e) - } - - j++ - - } - - if g, e := j, len(up); g != e { - t.Fatal(i, j, g, e) - } - } - - } -} - -func TestEnumeratorPrev(t *testing.T) { - // seeking within 3 keys: 10, 20, 30 - table := []struct { - k int - hit bool - keys []int - }{ - {5, false, []int{10}}, - {10, true, []int{10}}, - {15, false, []int{20, 10}}, - {20, true, []int{20, 10}}, - {25, false, []int{30, 20, 10}}, - {30, true, []int{30, 20, 10}}, - {35, false, []int{}}, - } - - for i, test := range table { - dn := test.keys - r := TreeNew(cmp) - - r.Set(10, 100) - r.Set(20, 200) - r.Set(30, 300) - - for verChange := 0; verChange < 16; verChange++ { - en, hit := r.Seek(test.k) - - if g, e := hit, test.hit; g != e { - t.Fatal(i, g, e) - } - - j := 0 - for { - if verChange&(1<= len(dn) { - t.Fatal(i, j, verChange) - } - - if g, e := k.(int), dn[j]; g != e { - t.Fatal(i, j, verChange, g, e) - } - - if g, e := v.(int), 10*dn[j]; g != e { - t.Fatal(i, g, e) - } - - j++ - - } - - if g, e := j, len(dn); g != e { - t.Fatal(i, j, g, e) - } - } - - } -} - -func BenchmarkSeekSeq1e3(b *testing.B) { - benchmarkSeekSeq(b, 1e3) -} - -func BenchmarkSeekSeq1e4(b *testing.B) { - benchmarkSeekSeq(b, 1e4) -} - -func BenchmarkSeekSeq1e5(b *testing.B) { - benchmarkSeekSeq(b, 1e5) -} - -func BenchmarkSeekSeq1e6(b *testing.B) { - benchmarkSeekSeq(b, 1e6) -} - -func benchmarkSeekSeq(b *testing.B, n int) { - for i := 0; i < b.N; i++ { - b.StopTimer() - t := TreeNew(cmp) - for j := 0; j < n; j++ { - t.Set(j, 0) - } - debug.FreeOSMemory() - b.StartTimer() - for j := 0; j < n; j++ { - e, _ := t.Seek(j) - e.Close() - } - b.StopTimer() - t.Close() - } - b.StopTimer() -} - -func BenchmarkSeekRnd1e3(b *testing.B) { - benchmarkSeekRnd(b, 1e3) -} - -func BenchmarkSeekRnd1e4(b *testing.B) { - benchmarkSeekRnd(b, 1e4) -} - -func BenchmarkSeekRnd1e5(b *testing.B) { - benchmarkSeekRnd(b, 1e5) -} - -func BenchmarkSeekRnd1e6(b *testing.B) { - benchmarkSeekRnd(b, 1e6) -} - -func benchmarkSeekRnd(b *testing.B, n int) { - r := TreeNew(cmp) - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - for _, v := range a { - r.Set(v, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, v := range a { - e, _ := r.Seek(v) - e.Close() - } - } - b.StopTimer() - r.Close() -} - -func BenchmarkNext1e3(b *testing.B) { - benchmarkNext(b, 1e3) -} - -func BenchmarkNext1e4(b *testing.B) { - benchmarkNext(b, 1e4) -} - -func BenchmarkNext1e5(b *testing.B) { - benchmarkNext(b, 1e5) -} - -func BenchmarkNext1e6(b *testing.B) { - benchmarkNext(b, 1e6) -} - -func benchmarkNext(b *testing.B, n int) { - t := TreeNew(cmp) - for i := 0; i < n; i++ { - t.Set(i, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - en, err := t.SeekFirst() - if err != nil { - b.Fatal(err) - } - - m := 0 - for { - if _, _, err = en.Next(); err != nil { - break - } - m++ - } - if m != n { - b.Fatal(m) - } - } - b.StopTimer() - t.Close() -} - -func BenchmarkPrev1e3(b *testing.B) { - benchmarkPrev(b, 1e3) -} - -func BenchmarkPrev1e4(b *testing.B) { - benchmarkPrev(b, 1e4) -} - -func BenchmarkPrev1e5(b *testing.B) { - benchmarkPrev(b, 1e5) -} - -func BenchmarkPrev1e6(b *testing.B) { - benchmarkPrev(b, 1e6) -} - -func benchmarkPrev(b *testing.B, n int) { - t := TreeNew(cmp) - for i := 0; i < n; i++ { - t.Set(i, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - en, err := t.SeekLast() - if err != nil { - b.Fatal(err) - } - - m := 0 - for { - if _, _, err = en.Prev(); err != nil { - break - } - m++ - } - if m != n { - b.Fatal(m) - } - } -} - -func TestSeekFirst0(t *testing.T) { - b := TreeNew(cmp) - _, err := b.SeekFirst() - if g, e := err, io.EOF; g != e { - t.Fatal(g, e) - } -} - -func TestSeekFirst1(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - en, err := b.SeekFirst() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Next() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekFirst2(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - b.Set(2, 20) - en, err := b.SeekFirst() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Next() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekFirst3(t *testing.T) { - b := TreeNew(cmp) - b.Set(2, 20) - b.Set(3, 30) - b.Set(1, 10) - en, err := b.SeekFirst() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Next() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if k != 3 || v != 30 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekLast0(t *testing.T) { - b := TreeNew(cmp) - _, err := b.SeekLast() - if g, e := err, io.EOF; g != e { - t.Fatal(g, e) - } -} - -func TestSeekLast1(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - en, err := b.SeekLast() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Prev() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekLast2(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - b.Set(2, 20) - en, err := b.SeekLast() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Prev() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekLast3(t *testing.T) { - b := TreeNew(cmp) - b.Set(2, 20) - b.Set(3, 30) - b.Set(1, 10) - en, err := b.SeekLast() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Prev() - if k != 3 || v != 30 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestPut(t *testing.T) { - tab := []struct { - pre []int // even index: K, odd index: V - newK int // Put(newK, ... - oldV int // Put()->oldV - exists bool // upd(exists) - write bool // upd()->write - post []int // even index: K, odd index: V - }{ - // 0 - { - []int{}, - 1, 0, false, false, - []int{}, - }, - { - []int{}, - 1, 0, false, true, - []int{1, -1}, - }, - { - []int{1, 10}, - 0, 0, false, false, - []int{1, 10}, - }, - { - []int{1, 10}, - 0, 0, false, true, - []int{0, -1, 1, 10}, - }, - { - []int{1, 10}, - 1, 10, true, false, - []int{1, 10}, - }, - - // 5 - { - []int{1, 10}, - 1, 10, true, true, - []int{1, -1}, - }, - { - []int{1, 10}, - 2, 0, false, false, - []int{1, 10}, - }, - { - []int{1, 10}, - 2, 0, false, true, - []int{1, 10, 2, -1}, - }, - } - - for iTest, test := range tab { - tr := TreeNew(cmp) - for i := 0; i < len(test.pre); i += 2 { - k, v := test.pre[i], test.pre[i+1] - tr.Set(k, v) - } - - oldV, written := tr.Put(test.newK, func(old interface{}, exists bool) (newV interface{}, write bool) { - if g, e := exists, test.exists; g != e { - t.Fatal(iTest, g, e) - } - - if exists { - if g, e := old.(int), test.oldV; g != e { - t.Fatal(iTest, g, e) - } - } - return -1, test.write - }) - if test.exists { - if g, e := oldV.(int), test.oldV; g != e { - t.Fatal(iTest, g, e) - } - } - - if g, e := written, test.write; g != e { - t.Fatal(iTest, g, e) - } - - n := len(test.post) - en, err := tr.SeekFirst() - if err != nil { - if n == 0 && err == io.EOF { - continue - } - - t.Fatal(iTest, err) - } - - for i := 0; i < len(test.post); i += 2 { - k, v, err := en.Next() - if err != nil { - t.Fatal(iTest, err) - } - - if g, e := k.(int), test.post[i]; g != e { - t.Fatal(iTest, g, e) - } - - if g, e := v.(int), test.post[i+1]; g != e { - t.Fatal(iTest, g, e) - } - } - - _, _, err = en.Next() - if g, e := err, io.EOF; g != e { - t.Fatal(iTest, g, e) - } - } -} - -func TestSeek(t *testing.T) { - const N = 1 << 13 - tr := TreeNew(cmp) - for i := 0; i < N; i++ { - k := 2*i + 1 - tr.Set(k, nil) - } - for i := 0; i < N; i++ { - k := 2 * i - e, ok := tr.Seek(k) - if ok { - t.Fatal(k) - } - - for j := i; j < N; j++ { - k2, _, err := e.Next() - if err != nil { - t.Fatal(k, err) - } - - if g, e := k2, 2*j+1; g != e { - t.Fatal(j, g, e) - } - } - - _, _, err := e.Next() - if err != io.EOF { - t.Fatalf("expected io.EOF, got %v", err) - } - } -} - -func TestPR4(t *testing.T) { - tr := TreeNew(cmp) - for i := 0; i < 2*kd+1; i++ { - k := 1000 * i - tr.Set(k, nil) - } - tr.Delete(1000 * kd) - for i := 0; i < kd; i++ { - tr.Set(1000*(kd+1)-1-i, nil) - } - k := 1000*(kd+1) - 1 - kd - tr.Set(k, nil) - if _, ok := tr.Get(k); !ok { - t.Fatalf("key lost: %v", k) - } -} diff --git a/vendor/github.com/cznic/b/btree.go b/vendor/github.com/cznic/b/btree.go deleted file mode 100644 index 0809a825..00000000 --- a/vendor/github.com/cznic/b/btree.go +++ /dev/null @@ -1,929 +0,0 @@ -// Copyright 2014 The b Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package b - -import ( - "fmt" - "io" - "sync" -) - -const ( - kx = 32 //TODO benchmark tune this number if using custom key/value type(s). - kd = 32 //TODO benchmark tune this number if using custom key/value type(s). -) - -func init() { - if kd < 1 { - panic(fmt.Errorf("kd %d: out of range", kd)) - } - - if kx < 2 { - panic(fmt.Errorf("kx %d: out of range", kx)) - } -} - -var ( - btDPool = sync.Pool{New: func() interface{} { return &d{} }} - btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}} - btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}} - btXPool = sync.Pool{New: func() interface{} { return &x{} }} -) - -type btTpool struct{ sync.Pool } - -func (p *btTpool) get(cmp Cmp) *Tree { - x := p.Get().(*Tree) - x.cmp = cmp - return x -} - -type btEpool struct{ sync.Pool } - -func (p *btEpool) get(err error, hit bool, i int, k interface{} /*K*/, q *d, t *Tree, ver int64) *Enumerator { - x := p.Get().(*Enumerator) - x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver - return x -} - -type ( - // Cmp compares a and b. Return value is: - // - // < 0 if a < b - // 0 if a == b - // > 0 if a > b - // - Cmp func(a, b interface{} /*K*/) int - - d struct { // data page - c int - d [2*kd + 1]de - n *d - p *d - } - - de struct { // d element - k interface{} /*K*/ - v interface{} /*V*/ - } - - // Enumerator captures the state of enumerating a tree. It is returned - // from the Seek* methods. The enumerator is aware of any mutations - // made to the tree in the process of enumerating it and automatically - // resumes the enumeration at the proper key, if possible. - // - // However, once an Enumerator returns io.EOF to signal "no more - // items", it does no more attempt to "resync" on tree mutation(s). In - // other words, io.EOF from an Enumerator is "sticky" (idempotent). - Enumerator struct { - err error - hit bool - i int - k interface{} /*K*/ - q *d - t *Tree - ver int64 - } - - // Tree is a B+tree. - Tree struct { - c int - cmp Cmp - first *d - last *d - r interface{} - ver int64 - } - - xe struct { // x element - ch interface{} - k interface{} /*K*/ - } - - x struct { // index page - c int - x [2*kx + 2]xe - } -) - -var ( // R/O zero values - zd d - zde de - ze Enumerator - zk interface{} /*K*/ - zt Tree - zx x - zxe xe -) - -func clr(q interface{}) { - switch x := q.(type) { - case *x: - for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn - clr(x.x[i].ch) - } - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } -} - -// -------------------------------------------------------------------------- x - -func newX(ch0 interface{}) *x { - r := btXPool.Get().(*x) - r.x[0].ch = ch0 - return r -} - -func (q *x) extract(i int) { - q.c-- - if i < q.c { - copy(q.x[i:], q.x[i+1:q.c+1]) - q.x[q.c].ch = q.x[q.c+1].ch - q.x[q.c].k = zk // GC - q.x[q.c+1] = zxe // GC - } -} - -func (q *x) insert(i int, k interface{} /*K*/, ch interface{}) *x { - c := q.c - if i < c { - q.x[c+1].ch = q.x[c].ch - copy(q.x[i+2:], q.x[i+1:c]) - q.x[i+1].k = q.x[i].k - } - c++ - q.c = c - q.x[i].k = k - q.x[i+1].ch = ch - return q -} - -func (q *x) siblings(i int) (l, r *d) { - if i >= 0 { - if i > 0 { - l = q.x[i-1].ch.(*d) - } - if i < q.c { - r = q.x[i+1].ch.(*d) - } - } - return -} - -// -------------------------------------------------------------------------- d - -func (l *d) mvL(r *d, c int) { - copy(l.d[l.c:], r.d[:c]) - copy(r.d[:], r.d[c:r.c]) - l.c += c - r.c -= c -} - -func (l *d) mvR(r *d, c int) { - copy(r.d[c:], r.d[:r.c]) - copy(r.d[:c], l.d[l.c-c:]) - r.c += c - l.c -= c -} - -// ----------------------------------------------------------------------- Tree - -// TreeNew returns a newly created, empty Tree. The compare function is used -// for key collation. -func TreeNew(cmp Cmp) *Tree { - return btTPool.get(cmp) -} - -// Clear removes all K/V pairs from the tree. -func (t *Tree) Clear() { - if t.r == nil { - return - } - - clr(t.r) - t.c, t.first, t.last, t.r = 0, nil, nil, nil - t.ver++ -} - -// Close performs Clear and recycles t to a pool for possible later reuse. No -// references to t should exist or such references must not be used afterwards. -func (t *Tree) Close() { - t.Clear() - *t = zt - btTPool.Put(t) -} - -func (t *Tree) cat(p *x, q, r *d, pi int) { - t.ver++ - q.mvL(r, r.c) - if r.n != nil { - r.n.p = q - } else { - t.last = q - } - q.n = r.n - *r = zd - btDPool.Put(r) - if p.c > 1 { - p.extract(pi) - p.x[pi].ch = q - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -func (t *Tree) catX(p, q, r *x, pi int) { - t.ver++ - q.x[q.c].k = p.x[pi].k - copy(q.x[q.c+1:], r.x[:r.c]) - q.c += r.c + 1 - q.x[q.c].ch = r.x[r.c].ch - *r = zx - btXPool.Put(r) - if p.c > 1 { - p.c-- - pc := p.c - if pi < pc { - p.x[pi].k = p.x[pi+1].k - copy(p.x[pi+1:], p.x[pi+2:pc+1]) - p.x[pc].ch = p.x[pc+1].ch - p.x[pc].k = zk // GC - p.x[pc+1].ch = nil // GC - } - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -// Delete removes the k's KV pair, if it exists, in which case Delete returns -// true. -func (t *Tree) Delete(k interface{} /*K*/) (ok bool) { - pi := -1 - var p *x - q := t.r - if q == nil { - return false - } - - for { - var i int - i, ok = t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[pi].ch - ok = false - continue - case *d: - t.extract(x, i) - if x.c >= kd { - return true - } - - if q != t.r { - t.underflow(p, x, pi) - } else if t.c == 0 { - t.Clear() - } - return true - } - } - - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - return false - } - } -} - -func (t *Tree) extract(q *d, i int) { // (r interface{} /*V*/) { - t.ver++ - //r = q.d[i].v // prepared for Extract - q.c-- - if i < q.c { - copy(q.d[i:], q.d[i+1:q.c+1]) - } - q.d[q.c] = zde // GC - t.c-- - return -} - -func (t *Tree) find(q interface{}, k interface{} /*K*/) (i int, ok bool) { - var mk interface{} /*K*/ - l := 0 - switch x := q.(type) { - case *x: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.x[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - case *d: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.d[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - } - return l, false -} - -// First returns the first item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *Tree) First() (k interface{} /*K*/, v interface{} /*V*/) { - if q := t.first; q != nil { - q := &q.d[0] - k, v = q.k, q.v - } - return -} - -// Get returns the value associated with k and true if it exists. Otherwise Get -// returns (zero-value, false). -func (t *Tree) Get(k interface{} /*K*/) (v interface{} /*V*/, ok bool) { - q := t.r - if q == nil { - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return x.d[i].v, true - } - } - switch x := q.(type) { - case *x: - q = x.x[i].ch - default: - return - } - } -} - -func (t *Tree) insert(q *d, i int, k interface{} /*K*/, v interface{} /*V*/) *d { - t.ver++ - c := q.c - if i < c { - copy(q.d[i+1:], q.d[i:c]) - } - c++ - q.c = c - q.d[i].k, q.d[i].v = k, v - t.c++ - return q -} - -// Last returns the last item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *Tree) Last() (k interface{} /*K*/, v interface{} /*V*/) { - if q := t.last; q != nil { - q := &q.d[q.c-1] - k, v = q.k, q.v - } - return -} - -// Len returns the number of items in the tree. -func (t *Tree) Len() int { - return t.c -} - -func (t *Tree) overflow(p *x, q *d, pi, i int, k interface{} /*K*/, v interface{} /*V*/) { - t.ver++ - l, r := p.siblings(pi) - - if l != nil && l.c < 2*kd && i != 0 { - l.mvL(q, 1) - t.insert(q, i-1, k, v) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && r.c < 2*kd { - if i < 2*kd { - q.mvR(r, 1) - t.insert(q, i, k, v) - p.x[pi].k = r.d[0].k - return - } - - t.insert(r, 0, k, v) - p.x[pi].k = k - return - } - - t.split(p, q, pi, i, k, v) -} - -// Seek returns an Enumerator positioned on an item such that k >= item's key. -// ok reports if k == item.key The Enumerator's position is possibly after the -// last item in the tree. -func (t *Tree) Seek(k interface{} /*K*/) (e *Enumerator, ok bool) { - q := t.r - if q == nil { - e = btEPool.get(nil, false, 0, k, nil, t, t.ver) - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), true - } - } - - switch x := q.(type) { - case *x: - q = x.x[i].ch - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), false - } - } -} - -// SeekFirst returns an enumerator positioned on the first KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekFirst() (e *Enumerator, err error) { - q := t.first - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, 0, q.d[0].k, q, t, t.ver), nil -} - -// SeekLast returns an enumerator positioned on the last KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekLast() (e *Enumerator, err error) { - q := t.last - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, q.c-1, q.d[q.c-1].k, q, t, t.ver), nil -} - -// Set sets the value associated with k. -func (t *Tree) Set(k interface{} /*K*/, v interface{} /*V*/) { - //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) - //defer func() { - // dbg("--- POST\n%s\n====\n", t.dump()) - //}() - - pi := -1 - var p *x - q := t.r - if q == nil { - z := t.insert(btDPool.Get().(*d), 0, k, v) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[i+1].ch - continue - case *d: - x.d[i].v = v - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - switch { - case x.c < 2*kd: - t.insert(x, i, k, v) - default: - t.overflow(p, x, pi, i, k, v) - } - return - } - } -} - -// Put combines Get and Set in a more efficient way where the tree is walked -// only once. The upd(ater) receives (old-value, true) if a KV pair for k -// exists or (zero-value, false) otherwise. It can then return a (new-value, -// true) to create or overwrite the existing value in the KV pair, or -// (whatever, false) if it decides not to create or not to update the value of -// the KV pair. -// -// tree.Set(k, v) call conceptually equals calling -// -// tree.Put(k, func(interface{} /*K*/, bool){ return v, true }) -// -// modulo the differing return values. -func (t *Tree) Put(k interface{} /*K*/, upd func(oldV interface{} /*V*/, exists bool) (newV interface{} /*V*/, write bool)) (oldV interface{} /*V*/, written bool) { - pi := -1 - var p *x - q := t.r - var newV interface{} /*V*/ - if q == nil { - // new KV pair in empty tree - newV, written = upd(newV, false) - if !written { - return - } - - z := t.insert(btDPool.Get().(*d), 0, k, newV) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[i+1].ch - continue - case *d: - oldV = x.d[i].v - newV, written = upd(oldV, true) - if !written { - return - } - - x.d[i].v = newV - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: // new KV pair - newV, written = upd(newV, false) - if !written { - return - } - - switch { - case x.c < 2*kd: - t.insert(x, i, k, newV) - default: - t.overflow(p, x, pi, i, k, newV) - } - return - } - } -} - -func (t *Tree) split(p *x, q *d, pi, i int, k interface{} /*K*/, v interface{} /*V*/) { - t.ver++ - r := btDPool.Get().(*d) - if q.n != nil { - r.n = q.n - r.n.p = r - } else { - t.last = r - } - q.n = r - r.p = q - - copy(r.d[:], q.d[kd:2*kd]) - for i := range q.d[kd:] { - q.d[kd+i] = zde - } - q.c = kd - r.c = kd - var done bool - if i > kd { - done = true - t.insert(r, i-kd, k, v) - } - if pi >= 0 { - p.insert(pi, r.d[0].k, r) - } else { - t.r = newX(q).insert(0, r.d[0].k, r) - } - if done { - return - } - - t.insert(q, i, k, v) -} - -func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - r := btXPool.Get().(*x) - copy(r.x[:], q.x[kx+1:]) - q.c = kx - r.c = kx - if pi >= 0 { - p.insert(pi, q.x[kx].k, r) - q.x[kx].k = zk - for i := range q.x[kx+1:] { - q.x[kx+i+1] = zxe - } - - switch { - case i < kx: - return q, i - case i == kx: - return p, pi - default: // i > kx - return r, i - kx - 1 - } - } - - nr := newX(q).insert(0, q.x[kx].k, r) - t.r = nr - q.x[kx].k = zk - for i := range q.x[kx+1:] { - q.x[kx+i+1] = zxe - } - - switch { - case i < kx: - return q, i - case i == kx: - return nr, 0 - default: // i > kx - return r, i - kx - 1 - } -} - -func (t *Tree) underflow(p *x, q *d, pi int) { - t.ver++ - l, r := p.siblings(pi) - - if l != nil && l.c+q.c >= 2*kd { - l.mvR(q, 1) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && q.c+r.c >= 2*kd { - q.mvL(r, 1) - p.x[pi].k = r.d[0].k - r.d[r.c] = zde // GC - return - } - - if l != nil { - t.cat(p, l, q, pi-1) - return - } - - t.cat(p, q, r, pi) -} - -func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - var l, r *x - - if pi >= 0 { - if pi > 0 { - l = p.x[pi-1].ch.(*x) - } - if pi < p.c { - r = p.x[pi+1].ch.(*x) - } - } - - if l != nil && l.c > kx { - q.x[q.c+1].ch = q.x[q.c].ch - copy(q.x[1:], q.x[:q.c]) - q.x[0].ch = l.x[l.c].ch - q.x[0].k = p.x[pi-1].k - q.c++ - i++ - l.c-- - p.x[pi-1].k = l.x[l.c].k - return q, i - } - - if r != nil && r.c > kx { - q.x[q.c].k = p.x[pi].k - q.c++ - q.x[q.c].ch = r.x[0].ch - p.x[pi].k = r.x[0].k - copy(r.x[:], r.x[1:r.c]) - r.c-- - rc := r.c - r.x[rc].ch = r.x[rc+1].ch - r.x[rc].k = zk - r.x[rc+1].ch = nil - return q, i - } - - if l != nil { - i += l.c + 1 - t.catX(p, l, q, pi-1) - q = l - return q, i - } - - t.catX(p, q, r, pi) - return q, i -} - -// ----------------------------------------------------------------- Enumerator - -// Close recycles e to a pool for possible later reuse. No references to e -// should exist or such references must not be used afterwards. -func (e *Enumerator) Close() { - *e = ze - btEPool.Put(e) -} - -// Next returns the currently enumerated item, if it exists and moves to the -// next item in the key collation order. If there is no item to return, err == -// io.EOF is returned. -func (e *Enumerator) Next() (k interface{} /*K*/, v interface{} /*V*/, err error) { - if err = e.err; err != nil { - return - } - - if e.ver != e.t.ver { - f, hit := e.t.Seek(e.k) - if !e.hit && hit { - if err = f.next(); err != nil { - return - } - } - - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return - } - - if e.i >= e.q.c { - if err = e.next(); err != nil { - return - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, false - e.next() - return -} - -func (e *Enumerator) next() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i < e.q.c-1: - e.i++ - default: - if e.q, e.i = e.q.n, 0; e.q == nil { - e.err = io.EOF - } - } - return e.err -} - -// Prev returns the currently enumerated item, if it exists and moves to the -// previous item in the key collation order. If there is no item to return, err -// == io.EOF is returned. -func (e *Enumerator) Prev() (k interface{} /*K*/, v interface{} /*V*/, err error) { - if err = e.err; err != nil { - return - } - - if e.ver != e.t.ver { - f, hit := e.t.Seek(e.k) - if !e.hit && hit { - if err = f.prev(); err != nil { - return - } - } - - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return - } - - if e.i >= e.q.c { - if err = e.next(); err != nil { - return - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, false - e.prev() - return -} - -func (e *Enumerator) prev() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i > 0: - e.i-- - default: - if e.q = e.q.p; e.q == nil { - e.err = io.EOF - break - } - - e.i = e.q.c - 1 - } - return e.err -} diff --git a/vendor/github.com/cznic/b/doc.go b/vendor/github.com/cznic/b/doc.go deleted file mode 100644 index ddcf7370..00000000 --- a/vendor/github.com/cznic/b/doc.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The b Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package b implements the B+tree flavor of a BTree. -// -// Changelog -// -// 2014-06-26: Lower GC presure by recycling things. -// -// 2014-04-18: Added new method Put. -// -// Generic types -// -// Keys and their associated values are interface{} typed, similar to all of -// the containers in the standard library. -// -// Semiautomatic production of a type specific variant of this package is -// supported via -// -// $ make generic -// -// This command will write to stdout a version of the btree.go file where every -// key type occurrence is replaced by the word 'KEY' and every value type -// occurrence is replaced by the word 'VALUE'. Then you have to replace these -// tokens with your desired type(s), using any technique you're comfortable -// with. -// -// This is how, for example, 'example/int.go' was created: -// -// $ mkdir example -// $ make generic | sed -e 's/KEY/int/g' -e 's/VALUE/int/g' > example/int.go -// -// No other changes to int.go are necessary, it compiles just fine. -// -// Running the benchmarks for 1000 keys on a machine with Intel i5-4670 CPU @ -// 3.4GHz, Go release 1.4.2. -// -// $ go test -bench 1e3 example/all_test.go example/int.go -// PASS -// BenchmarkSetSeq1e3 10000 151620 ns/op -// BenchmarkGetSeq1e3 10000 115354 ns/op -// BenchmarkSetRnd1e3 5000 255865 ns/op -// BenchmarkGetRnd1e3 10000 140466 ns/op -// BenchmarkDelSeq1e3 10000 143860 ns/op -// BenchmarkDelRnd1e3 10000 188228 ns/op -// BenchmarkSeekSeq1e3 10000 156448 ns/op -// BenchmarkSeekRnd1e3 10000 190587 ns/op -// BenchmarkNext1e3 200000 9407 ns/op -// BenchmarkPrev1e3 200000 9306 ns/op -// ok command-line-arguments 26.369s -// $ -package b diff --git a/vendor/github.com/cznic/b/example/Makefile b/vendor/github.com/cznic/b/example/Makefile deleted file mode 100644 index 6c0f1856..00000000 --- a/vendor/github.com/cznic/b/example/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2014 The b Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -.PHONY: all todo clean cover mem - -testbin=b.test - -all: editor - go build - go vet - make todo - -editor: - go fmt - go test -i - go test - -mem: - go test -c - ./$(testbin) -test.bench . -test.memprofile mem.out -test.memprofilerate 1 - go tool pprof --lines --web --alloc_space $(testbin) mem.out - -todo: - @grep -n ^[[:space:]]*_[[:space:]]*=[[:space:]][[:alpha:]][[:alnum:]]* *.go || true - @grep -n TODO *.go || true - @grep -n BUG *.go || true - @grep -n println *.go || true - -clean: - @go clean - rm -f *~ - -cover: - t=$(shell tempfile) ; go test -coverprofile $$t && go tool cover -html $$t && unlink $$t diff --git a/vendor/github.com/cznic/b/example/all_test.go b/vendor/github.com/cznic/b/example/all_test.go deleted file mode 100644 index cfc7848c..00000000 --- a/vendor/github.com/cznic/b/example/all_test.go +++ /dev/null @@ -1,1126 +0,0 @@ -// Copyright 2014 The b Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package b - -import ( - "io" - "math" - "runtime/debug" - "testing" - - "github.com/cznic/fileutil" - "github.com/cznic/mathutil" -) - -func rng() *mathutil.FC32 { - x, err := mathutil.NewFC32(math.MinInt32/4, math.MaxInt32/4, false) - if err != nil { - panic(err) - } - - return x -} - -func cmp(a, b int) int { - return a - b -} - -func TestGet0(t *testing.T) { - r := TreeNew(cmp) - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - _, ok := r.Get(42) - if ok { - t.Fatal(ok) - } - -} - -func TestSetGet0(t *testing.T) { - r := TreeNew(cmp) - set := r.Set - set(42, 314) - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - v, ok := r.Get(42) - if !ok { - t.Fatal(ok) - } - - if g, e := v, 314; g != e { - t.Fatal(g, e) - } - - set(42, 278) - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - v, ok = r.Get(42) - if !ok { - t.Fatal(ok) - } - - if g, e := v, 278; g != e { - t.Fatal(g, e) - } - - set(420, 50) - if g, e := r.Len(), 2; g != e { - t.Fatal(g, e) - } - - v, ok = r.Get(42) - if !ok { - t.Fatal(ok) - } - - if g, e := v, 278; g != e { - t.Fatal(g, e) - } - - v, ok = r.Get(420) - if !ok { - t.Fatal(ok) - } - - if g, e := v, 50; g != e { - t.Fatal(g, e) - } -} - -func TestSetGet1(t *testing.T) { - const N = 90000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x314259} { - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - for i := range a { - a[i] = (i ^ x) << 1 - } - for i, k := range a { - set(k, k^x) - if g, e := r.Len(), i+1; g != e { - t.Fatal(i, g, e) - } - } - - for i, k := range a { - v, ok := r.Get(k) - if !ok { - t.Fatal(i, k, v, ok) - } - - if g, e := v, k^x; g != e { - t.Fatal(i, g, e) - } - - k |= 1 - _, ok = r.Get(k) - if ok { - t.Fatal(i, k) - } - - } - } -} - -func BenchmarkSetSeq1e3(b *testing.B) { - benchmarkSetSeq(b, 1e3) -} - -func BenchmarkSetSeq1e4(b *testing.B) { - benchmarkSetSeq(b, 1e4) -} - -func BenchmarkSetSeq1e5(b *testing.B) { - benchmarkSetSeq(b, 1e5) -} - -func BenchmarkSetSeq1e6(b *testing.B) { - benchmarkSetSeq(b, 1e6) -} - -func benchmarkSetSeq(b *testing.B, n int) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - debug.FreeOSMemory() - b.StartTimer() - for j := 0; j < n; j++ { - r.Set(j, j) - } - b.StopTimer() - r.Close() - } - b.StopTimer() -} - -func BenchmarkGetSeq1e3(b *testing.B) { - benchmarkGetSeq(b, 1e3) -} - -func BenchmarkGetSeq1e4(b *testing.B) { - benchmarkGetSeq(b, 1e4) -} - -func BenchmarkGetSeq1e5(b *testing.B) { - benchmarkGetSeq(b, 1e5) -} - -func BenchmarkGetSeq1e6(b *testing.B) { - benchmarkGetSeq(b, 1e6) -} - -func benchmarkGetSeq(b *testing.B, n int) { - r := TreeNew(cmp) - for i := 0; i < n; i++ { - r.Set(i, i) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < n; j++ { - r.Get(j) - } - } - b.StopTimer() - r.Close() -} - -func BenchmarkSetRnd1e3(b *testing.B) { - benchmarkSetRnd(b, 1e3) -} - -func BenchmarkSetRnd1e4(b *testing.B) { - benchmarkSetRnd(b, 1e4) -} - -func BenchmarkSetRnd1e5(b *testing.B) { - benchmarkSetRnd(b, 1e5) -} - -func BenchmarkSetRnd1e6(b *testing.B) { - benchmarkSetRnd(b, 1e6) -} - -func benchmarkSetRnd(b *testing.B, n int) { - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - debug.FreeOSMemory() - b.StartTimer() - for _, v := range a { - r.Set(v, 0) - } - b.StopTimer() - r.Close() - } - b.StopTimer() -} - -func BenchmarkGetRnd1e3(b *testing.B) { - benchmarkGetRnd(b, 1e3) -} - -func BenchmarkGetRnd1e4(b *testing.B) { - benchmarkGetRnd(b, 1e4) -} - -func BenchmarkGetRnd1e5(b *testing.B) { - benchmarkGetRnd(b, 1e5) -} - -func BenchmarkGetRnd1e6(b *testing.B) { - benchmarkGetRnd(b, 1e6) -} - -func benchmarkGetRnd(b *testing.B, n int) { - r := TreeNew(cmp) - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - for _, v := range a { - r.Set(v, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, v := range a { - r.Get(v) - } - } - b.StopTimer() - r.Close() -} - -func TestSetGet2(t *testing.T) { - const N = 70000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x314259} { - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - rng := rng() - for i := range a { - a[i] = (rng.Next() ^ x) << 1 - } - for i, k := range a { - set(k, k^x) - if g, e := r.Len(), i+1; g != e { - t.Fatal(i, x, g, e) - } - } - - for i, k := range a { - v, ok := r.Get(k) - if !ok { - t.Fatal(i, k, v, ok) - } - - if g, e := v, k^x; g != e { - t.Fatal(i, g, e) - } - - k |= 1 - _, ok = r.Get(k) - if ok { - t.Fatal(i, k) - } - } - } -} - -func TestSetGet3(t *testing.T) { - r := TreeNew(cmp) - set := r.Set - var i int - for i = 0; ; i++ { - set(i, -i) - if _, ok := r.r.(*x); ok { - break - } - } - for j := 0; j <= i; j++ { - set(j, j) - } - - for j := 0; j <= i; j++ { - v, ok := r.Get(j) - if !ok { - t.Fatal(j) - } - - if g, e := v, j; g != e { - t.Fatal(g, e) - } - } -} - -func TestDelete0(t *testing.T) { - r := TreeNew(cmp) - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - r.Set(0, 0) - if ok := r.Delete(1); ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - r.Set(0, 0) - r.Set(1, 1) - if ok := r.Delete(1); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(1); ok { - t.Fatal(ok) - } - - if ok := r.Delete(0); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - r.Set(0, 0) - r.Set(1, 1) - if ok := r.Delete(0); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 1; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(0); ok { - t.Fatal(ok) - } - - if ok := r.Delete(1); !ok { - t.Fatal(ok) - } - - if g, e := r.Len(), 0; g != e { - t.Fatal(g, e) - } - - if ok := r.Delete(1); ok { - t.Fatal(ok) - } -} - -func TestDelete1(t *testing.T) { - const N = 100000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x314259} { - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - for i := range a { - a[i] = (i ^ x) << 1 - } - for _, k := range a { - set(k, 0) - } - - for i, k := range a { - ok := r.Delete(k) - if !ok { - t.Fatal(i, x, k) - } - - if g, e := r.Len(), N-i-1; g != e { - t.Fatal(i, g, e) - } - } - } -} - -func BenchmarkDelSeq1e3(b *testing.B) { - benchmarkDelSeq(b, 1e3) -} - -func BenchmarkDelSeq1e4(b *testing.B) { - benchmarkDelSeq(b, 1e4) -} - -func BenchmarkDelSeq1e5(b *testing.B) { - benchmarkDelSeq(b, 1e5) -} - -func BenchmarkDelSeq1e6(b *testing.B) { - benchmarkDelSeq(b, 1e6) -} - -func benchmarkDelSeq(b *testing.B, n int) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - for i := 0; i < n; i++ { - r.Set(i, i) - } - debug.FreeOSMemory() - b.StartTimer() - for j := 0; j < n; j++ { - r.Delete(j) - } - b.StopTimer() - r.Close() - } - b.StopTimer() -} - -func BenchmarkDelRnd1e3(b *testing.B) { - benchmarkDelRnd(b, 1e3) -} - -func BenchmarkDelRnd1e4(b *testing.B) { - benchmarkDelRnd(b, 1e4) -} - -func BenchmarkDelRnd1e5(b *testing.B) { - benchmarkDelRnd(b, 1e5) -} - -func BenchmarkDelRnd1e6(b *testing.B) { - benchmarkDelRnd(b, 1e6) -} - -func benchmarkDelRnd(b *testing.B, n int) { - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - r := TreeNew(cmp) - for _, v := range a { - r.Set(v, 0) - } - debug.FreeOSMemory() - b.StartTimer() - for _, v := range a { - r.Delete(v) - } - b.StopTimer() - r.Close() - } - b.StopTimer() -} - -func TestDelete2(t *testing.T) { - const N = 80000 - for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x314259} { - r := TreeNew(cmp) - set := r.Set - a := make([]int, N) - rng := rng() - for i := range a { - a[i] = (rng.Next() ^ x) << 1 - } - for _, k := range a { - set(k, 0) - } - - for i, k := range a { - ok := r.Delete(k) - if !ok { - t.Fatal(i, x, k) - } - - if g, e := r.Len(), N-i-1; g != e { - t.Fatal(i, g, e) - } - } - } -} - -func TestEnumeratorNext(t *testing.T) { - // seeking within 3 keys: 10, 20, 30 - table := []struct { - k int - hit bool - keys []int - }{ - {5, false, []int{10, 20, 30}}, - {10, true, []int{10, 20, 30}}, - {15, false, []int{20, 30}}, - {20, true, []int{20, 30}}, - {25, false, []int{30}}, - {30, true, []int{30}}, - {35, false, []int{}}, - } - - for i, test := range table { - up := test.keys - r := TreeNew(cmp) - - r.Set(10, 100) - r.Set(20, 200) - r.Set(30, 300) - - for verChange := 0; verChange < 16; verChange++ { - en, hit := r.Seek(test.k) - - if g, e := hit, test.hit; g != e { - t.Fatal(i, g, e) - } - - j := 0 - for { - if verChange&(1<= len(up) { - t.Fatal(i, j, verChange) - } - - if g, e := k, up[j]; g != e { - t.Fatal(i, j, verChange, g, e) - } - - if g, e := v, 10*up[j]; g != e { - t.Fatal(i, g, e) - } - - j++ - - } - - if g, e := j, len(up); g != e { - t.Fatal(i, j, g, e) - } - } - - } -} - -func TestEnumeratorPrev(t *testing.T) { - // seeking within 3 keys: 10, 20, 30 - table := []struct { - k int - hit bool - keys []int - }{ - {5, false, []int{10}}, - {10, true, []int{10}}, - {15, false, []int{20, 10}}, - {20, true, []int{20, 10}}, - {25, false, []int{30, 20, 10}}, - {30, true, []int{30, 20, 10}}, - {35, false, []int{}}, - } - - for i, test := range table { - dn := test.keys - r := TreeNew(cmp) - - r.Set(10, 100) - r.Set(20, 200) - r.Set(30, 300) - - for verChange := 0; verChange < 16; verChange++ { - en, hit := r.Seek(test.k) - - if g, e := hit, test.hit; g != e { - t.Fatal(i, g, e) - } - - j := 0 - for { - if verChange&(1<= len(dn) { - t.Fatal(i, j, verChange) - } - - if g, e := k, dn[j]; g != e { - t.Fatal(i, j, verChange, g, e) - } - - if g, e := v, 10*dn[j]; g != e { - t.Fatal(i, g, e) - } - - j++ - - } - - if g, e := j, len(dn); g != e { - t.Fatal(i, j, g, e) - } - } - - } -} - -func BenchmarkSeekSeq1e3(b *testing.B) { - benchmarkSeekSeq(b, 1e3) -} - -func BenchmarkSeekSeq1e4(b *testing.B) { - benchmarkSeekSeq(b, 1e4) -} - -func BenchmarkSeekSeq1e5(b *testing.B) { - benchmarkSeekSeq(b, 1e5) -} - -func BenchmarkSeekSeq1e6(b *testing.B) { - benchmarkSeekSeq(b, 1e6) -} - -func benchmarkSeekSeq(b *testing.B, n int) { - for i := 0; i < b.N; i++ { - b.StopTimer() - t := TreeNew(cmp) - for j := 0; j < n; j++ { - t.Set(j, 0) - } - debug.FreeOSMemory() - b.StartTimer() - for j := 0; j < n; j++ { - e, _ := t.Seek(j) - e.Close() - } - b.StopTimer() - t.Close() - } - b.StopTimer() -} - -func BenchmarkSeekRnd1e3(b *testing.B) { - benchmarkSeekRnd(b, 1e3) -} - -func BenchmarkSeekRnd1e4(b *testing.B) { - benchmarkSeekRnd(b, 1e4) -} - -func BenchmarkSeekRnd1e5(b *testing.B) { - benchmarkSeekRnd(b, 1e5) -} - -func BenchmarkSeekRnd1e6(b *testing.B) { - benchmarkSeekRnd(b, 1e6) -} - -func benchmarkSeekRnd(b *testing.B, n int) { - r := TreeNew(cmp) - rng := rng() - a := make([]int, n) - for i := range a { - a[i] = rng.Next() - } - for _, v := range a { - r.Set(v, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, v := range a { - e, _ := r.Seek(v) - e.Close() - } - } - b.StopTimer() - r.Close() -} - -func BenchmarkNext1e3(b *testing.B) { - benchmarkNext(b, 1e3) -} - -func BenchmarkNext1e4(b *testing.B) { - benchmarkNext(b, 1e4) -} - -func BenchmarkNext1e5(b *testing.B) { - benchmarkNext(b, 1e5) -} - -func BenchmarkNext1e6(b *testing.B) { - benchmarkNext(b, 1e6) -} - -func benchmarkNext(b *testing.B, n int) { - t := TreeNew(cmp) - for i := 0; i < n; i++ { - t.Set(i, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - en, err := t.SeekFirst() - if err != nil { - b.Fatal(err) - } - - m := 0 - for { - if _, _, err = en.Next(); err != nil { - break - } - m++ - } - if m != n { - b.Fatal(m) - } - } - b.StopTimer() - t.Close() -} - -func BenchmarkPrev1e3(b *testing.B) { - benchmarkPrev(b, 1e3) -} - -func BenchmarkPrev1e4(b *testing.B) { - benchmarkPrev(b, 1e4) -} - -func BenchmarkPrev1e5(b *testing.B) { - benchmarkPrev(b, 1e5) -} - -func BenchmarkPrev1e6(b *testing.B) { - benchmarkPrev(b, 1e6) -} - -func benchmarkPrev(b *testing.B, n int) { - t := TreeNew(cmp) - for i := 0; i < n; i++ { - t.Set(i, 0) - } - debug.FreeOSMemory() - b.ResetTimer() - for i := 0; i < b.N; i++ { - en, err := t.SeekLast() - if err != nil { - b.Fatal(err) - } - - m := 0 - for { - if _, _, err = en.Prev(); err != nil { - break - } - m++ - } - if m != n { - b.Fatal(m) - } - } - b.StopTimer() - t.Close() -} - -func TestSeekFirst0(t *testing.T) { - b := TreeNew(cmp) - _, err := b.SeekFirst() - if g, e := err, io.EOF; g != e { - t.Fatal(g, e) - } -} - -func TestSeekFirst1(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - en, err := b.SeekFirst() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Next() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekFirst2(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - b.Set(2, 20) - en, err := b.SeekFirst() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Next() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekFirst3(t *testing.T) { - b := TreeNew(cmp) - b.Set(2, 20) - b.Set(3, 30) - b.Set(1, 10) - en, err := b.SeekFirst() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Next() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if k != 3 || v != 30 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Next() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekLast0(t *testing.T) { - b := TreeNew(cmp) - _, err := b.SeekLast() - if g, e := err, io.EOF; g != e { - t.Fatal(g, e) - } -} - -func TestSeekLast1(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - en, err := b.SeekLast() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Prev() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekLast2(t *testing.T) { - b := TreeNew(cmp) - b.Set(1, 10) - b.Set(2, 20) - en, err := b.SeekLast() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Prev() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestSeekLast3(t *testing.T) { - b := TreeNew(cmp) - b.Set(2, 20) - b.Set(3, 30) - b.Set(1, 10) - en, err := b.SeekLast() - if err != nil { - t.Fatal(err) - } - - k, v, err := en.Prev() - if k != 3 || v != 30 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if k != 2 || v != 20 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if k != 1 || v != 10 || err != nil { - t.Fatal(k, v, err) - } - - k, v, err = en.Prev() - if err == nil { - t.Fatal(k, v, err) - } -} - -func TestPut(t *testing.T) { - tab := []struct { - pre []int // even index: K, odd index: V - newK int // Put(newK, ... - oldV int // Put()->oldV - exists bool // upd(exists) - write bool // upd()->write - post []int // even index: K, odd index: V - }{ - // 0 - { - []int{}, - 1, 0, false, false, - []int{}, - }, - { - []int{}, - 1, 0, false, true, - []int{1, -1}, - }, - { - []int{1, 10}, - 0, 0, false, false, - []int{1, 10}, - }, - { - []int{1, 10}, - 0, 0, false, true, - []int{0, -1, 1, 10}, - }, - { - []int{1, 10}, - 1, 10, true, false, - []int{1, 10}, - }, - - // 5 - { - []int{1, 10}, - 1, 10, true, true, - []int{1, -1}, - }, - { - []int{1, 10}, - 2, 0, false, false, - []int{1, 10}, - }, - { - []int{1, 10}, - 2, 0, false, true, - []int{1, 10, 2, -1}, - }, - } - - for iTest, test := range tab { - tr := TreeNew(cmp) - for i := 0; i < len(test.pre); i += 2 { - k, v := test.pre[i], test.pre[i+1] - tr.Set(k, v) - } - - oldV, written := tr.Put(test.newK, func(old int, exists bool) (newV int, write bool) { - if g, e := exists, test.exists; g != e { - t.Fatal(iTest, g, e) - } - - if exists { - if g, e := old, test.oldV; g != e { - t.Fatal(iTest, g, e) - } - } - return -1, test.write - }) - if test.exists { - if g, e := oldV, test.oldV; g != e { - t.Fatal(iTest, g, e) - } - } - - if g, e := written, test.write; g != e { - t.Fatal(iTest, g, e) - } - - n := len(test.post) - en, err := tr.SeekFirst() - if err != nil { - if n == 0 && err == io.EOF { - continue - } - - t.Fatal(iTest, err) - } - - for i := 0; i < len(test.post); i += 2 { - k, v, err := en.Next() - if err != nil { - t.Fatal(iTest, err) - } - - if g, e := k, test.post[i]; g != e { - t.Fatal(iTest, g, e) - } - - if g, e := v, test.post[i+1]; g != e { - t.Fatal(iTest, g, e) - } - } - - _, _, err = en.Next() - if g, e := err, io.EOF; g != e { - t.Fatal(iTest, g, e) - } - } -} diff --git a/vendor/github.com/cznic/b/example/int.go b/vendor/github.com/cznic/b/example/int.go deleted file mode 100644 index 54f7fdff..00000000 --- a/vendor/github.com/cznic/b/example/int.go +++ /dev/null @@ -1,929 +0,0 @@ -// Copyright 2014 The b Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package b - -import ( - "fmt" - "io" - "sync" -) - -const ( - kx = 32 //TODO benchmark tune this number if using custom key/value type(s). - kd = 32 //TODO benchmark tune this number if using custom key/value type(s). -) - -func init() { - if kd < 1 { - panic(fmt.Errorf("kd %d: out of range", kd)) - } - - if kx < 2 { - panic(fmt.Errorf("kx %d: out of range", kx)) - } -} - -var ( - btDPool = sync.Pool{New: func() interface{} { return &d{} }} - btEPool = btEpool{sync.Pool{New: func() interface{} { return &Enumerator{} }}} - btTPool = btTpool{sync.Pool{New: func() interface{} { return &Tree{} }}} - btXPool = sync.Pool{New: func() interface{} { return &x{} }} -) - -type btTpool struct{ sync.Pool } - -func (p *btTpool) get(cmp Cmp) *Tree { - x := p.Get().(*Tree) - x.cmp = cmp - return x -} - -type btEpool struct{ sync.Pool } - -func (p *btEpool) get(err error, hit bool, i int, k int, q *d, t *Tree, ver int64) *Enumerator { - x := p.Get().(*Enumerator) - x.err, x.hit, x.i, x.k, x.q, x.t, x.ver = err, hit, i, k, q, t, ver - return x -} - -type ( - // Cmp compares a and b. Return value is: - // - // < 0 if a < b - // 0 if a == b - // > 0 if a > b - // - Cmp func(a, b int) int - - d struct { // data page - c int - d [2*kd + 1]de - n *d - p *d - } - - de struct { // d element - k int - v int - } - - // Enumerator captures the state of enumerating a tree. It is returned - // from the Seek* methods. The enumerator is aware of any mutations - // made to the tree in the process of enumerating it and automatically - // resumes the enumeration at the proper key, if possible. - // - // However, once an Enumerator returns io.EOF to signal "no more - // items", it does no more attempt to "resync" on tree mutation(s). In - // other words, io.EOF from an Enumaretor is "sticky" (idempotent). - Enumerator struct { - err error - hit bool - i int - k int - q *d - t *Tree - ver int64 - } - - // Tree is a B+tree. - Tree struct { - c int - cmp Cmp - first *d - last *d - r interface{} - ver int64 - } - - xe struct { // x element - ch interface{} - k int - } - - x struct { // index page - c int - x [2*kx + 2]xe - } -) - -var ( // R/O zero values - zd d - zde de - ze Enumerator - zk int - zt Tree - zx x - zxe xe -) - -func clr(q interface{}) { - switch x := q.(type) { - case *x: - for i := 0; i <= x.c; i++ { // Ch0 Sep0 ... Chn-1 Sepn-1 Chn - clr(x.x[i].ch) - } - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } -} - -// -------------------------------------------------------------------------- x - -func newX(ch0 interface{}) *x { - r := btXPool.Get().(*x) - r.x[0].ch = ch0 - return r -} - -func (q *x) extract(i int) { - q.c-- - if i < q.c { - copy(q.x[i:], q.x[i+1:q.c+1]) - q.x[q.c].ch = q.x[q.c+1].ch - q.x[q.c].k = zk // GC - q.x[q.c+1] = zxe // GC - } -} - -func (q *x) insert(i int, k int, ch interface{}) *x { - c := q.c - if i < c { - q.x[c+1].ch = q.x[c].ch - copy(q.x[i+2:], q.x[i+1:c]) - q.x[i+1].k = q.x[i].k - } - c++ - q.c = c - q.x[i].k = k - q.x[i+1].ch = ch - return q -} - -func (q *x) siblings(i int) (l, r *d) { - if i >= 0 { - if i > 0 { - l = q.x[i-1].ch.(*d) - } - if i < q.c { - r = q.x[i+1].ch.(*d) - } - } - return -} - -// -------------------------------------------------------------------------- d - -func (l *d) mvL(r *d, c int) { - copy(l.d[l.c:], r.d[:c]) - copy(r.d[:], r.d[c:r.c]) - l.c += c - r.c -= c -} - -func (l *d) mvR(r *d, c int) { - copy(r.d[c:], r.d[:r.c]) - copy(r.d[:c], l.d[l.c-c:]) - r.c += c - l.c -= c -} - -// ----------------------------------------------------------------------- Tree - -// TreeNew returns a newly created, empty Tree. The compare function is used -// for key collation. -func TreeNew(cmp Cmp) *Tree { - return btTPool.get(cmp) -} - -// Clear removes all K/V pairs from the tree. -func (t *Tree) Clear() { - if t.r == nil { - return - } - - clr(t.r) - t.c, t.first, t.last, t.r = 0, nil, nil, nil - t.ver++ -} - -// Close performs Clear and recycles t to a pool for possible later reuse. No -// references to t should exist or such references must not be used afterwards. -func (t *Tree) Close() { - t.Clear() - *t = zt - btTPool.Put(t) -} - -func (t *Tree) cat(p *x, q, r *d, pi int) { - t.ver++ - q.mvL(r, r.c) - if r.n != nil { - r.n.p = q - } else { - t.last = q - } - q.n = r.n - *r = zd - btDPool.Put(r) - if p.c > 1 { - p.extract(pi) - p.x[pi].ch = q - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -func (t *Tree) catX(p, q, r *x, pi int) { - t.ver++ - q.x[q.c].k = p.x[pi].k - copy(q.x[q.c+1:], r.x[:r.c]) - q.c += r.c + 1 - q.x[q.c].ch = r.x[r.c].ch - *r = zx - btXPool.Put(r) - if p.c > 1 { - p.c-- - pc := p.c - if pi < pc { - p.x[pi].k = p.x[pi+1].k - copy(p.x[pi+1:], p.x[pi+2:pc+1]) - p.x[pc].ch = p.x[pc+1].ch - p.x[pc].k = zk // GC - p.x[pc+1].ch = nil // GC - } - return - } - - switch x := t.r.(type) { - case *x: - *x = zx - btXPool.Put(x) - case *d: - *x = zd - btDPool.Put(x) - } - t.r = q -} - -// Delete removes the k's KV pair, if it exists, in which case Delete returns -// true. -func (t *Tree) Delete(k int) (ok bool) { - pi := -1 - var p *x - q := t.r - if q == nil { - return false - } - - for { - var i int - i, ok = t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[pi].ch - ok = false - continue - case *d: - t.extract(x, i) - if x.c >= kd { - return true - } - - if q != t.r { - t.underflow(p, x, pi) - } else if t.c == 0 { - t.Clear() - } - return true - } - } - - switch x := q.(type) { - case *x: - if x.c < kx && q != t.r { - x, i = t.underflowX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - return false - } - } -} - -func (t *Tree) extract(q *d, i int) { // (r int) { - t.ver++ - //r = q.d[i].v // prepared for Extract - q.c-- - if i < q.c { - copy(q.d[i:], q.d[i+1:q.c+1]) - } - q.d[q.c] = zde // GC - t.c-- - return -} - -func (t *Tree) find(q interface{}, k int) (i int, ok bool) { - var mk int - l := 0 - switch x := q.(type) { - case *x: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.x[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - case *d: - h := x.c - 1 - for l <= h { - m := (l + h) >> 1 - mk = x.d[m].k - switch cmp := t.cmp(k, mk); { - case cmp > 0: - l = m + 1 - case cmp == 0: - return m, true - default: - h = m - 1 - } - } - } - return l, false -} - -// First returns the first item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *Tree) First() (k int, v int) { - if q := t.first; q != nil { - q := &q.d[0] - k, v = q.k, q.v - } - return -} - -// Get returns the value associated with k and true if it exists. Otherwise Get -// returns (zero-value, false). -func (t *Tree) Get(k int) (v int, ok bool) { - q := t.r - if q == nil { - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return x.d[i].v, true - } - } - switch x := q.(type) { - case *x: - q = x.x[i].ch - default: - return - } - } -} - -func (t *Tree) insert(q *d, i int, k int, v int) *d { - t.ver++ - c := q.c - if i < c { - copy(q.d[i+1:], q.d[i:c]) - } - c++ - q.c = c - q.d[i].k, q.d[i].v = k, v - t.c++ - return q -} - -// Last returns the last item of the tree in the key collating order, or -// (zero-value, zero-value) if the tree is empty. -func (t *Tree) Last() (k int, v int) { - if q := t.last; q != nil { - q := &q.d[q.c-1] - k, v = q.k, q.v - } - return -} - -// Len returns the number of items in the tree. -func (t *Tree) Len() int { - return t.c -} - -func (t *Tree) overflow(p *x, q *d, pi, i int, k int, v int) { - t.ver++ - l, r := p.siblings(pi) - - if l != nil && l.c < 2*kd { - l.mvL(q, 1) - t.insert(q, i-1, k, v) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && r.c < 2*kd { - if i < 2*kd { - q.mvR(r, 1) - t.insert(q, i, k, v) - p.x[pi].k = r.d[0].k - return - } - - t.insert(r, 0, k, v) - p.x[pi].k = k - return - } - - t.split(p, q, pi, i, k, v) -} - -// Seek returns an Enumerator positioned on a an item such that k >= item's -// key. ok reports if k == item.key The Enumerator's position is possibly -// after the last item in the tree. -func (t *Tree) Seek(k int) (e *Enumerator, ok bool) { - q := t.r - if q == nil { - e = btEPool.get(nil, false, 0, k, nil, t, t.ver) - return - } - - for { - var i int - if i, ok = t.find(q, k); ok { - switch x := q.(type) { - case *x: - q = x.x[i+1].ch - continue - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), true - } - } - - switch x := q.(type) { - case *x: - q = x.x[i].ch - case *d: - return btEPool.get(nil, ok, i, k, x, t, t.ver), false - } - } -} - -// SeekFirst returns an enumerator positioned on the first KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekFirst() (e *Enumerator, err error) { - q := t.first - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, 0, q.d[0].k, q, t, t.ver), nil -} - -// SeekLast returns an enumerator positioned on the last KV pair in the tree, -// if any. For an empty tree, err == io.EOF is returned and e will be nil. -func (t *Tree) SeekLast() (e *Enumerator, err error) { - q := t.last - if q == nil { - return nil, io.EOF - } - - return btEPool.get(nil, true, q.c-1, q.d[q.c-1].k, q, t, t.ver), nil -} - -// Set sets the value associated with k. -func (t *Tree) Set(k int, v int) { - //dbg("--- PRE Set(%v, %v)\n%s", k, v, t.dump()) - //defer func() { - // dbg("--- POST\n%s\n====\n", t.dump()) - //}() - - pi := -1 - var p *x - q := t.r - if q == nil { - z := t.insert(btDPool.Get().(*d), 0, k, v) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[i+1].ch - continue - case *d: - x.d[i].v = v - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: - switch { - case x.c < 2*kd: - t.insert(x, i, k, v) - default: - t.overflow(p, x, pi, i, k, v) - } - return - } - } -} - -// Put combines Get and Set in a more efficient way where the tree is walked -// only once. The upd(ater) receives (old-value, true) if a KV pair for k -// exists or (zero-value, false) otherwise. It can then return a (new-value, -// true) to create or overwrite the existing value in the KV pair, or -// (whatever, false) if it decides not to create or not to update the value of -// the KV pair. -// -// tree.Set(k, v) call conceptually equals calling -// -// tree.Put(k, func(int, bool){ return v, true }) -// -// modulo the differing return values. -func (t *Tree) Put(k int, upd func(oldV int, exists bool) (newV int, write bool)) (oldV int, written bool) { - pi := -1 - var p *x - q := t.r - var newV int - if q == nil { - // new KV pair in empty tree - newV, written = upd(newV, false) - if !written { - return - } - - z := t.insert(btDPool.Get().(*d), 0, k, newV) - t.r, t.first, t.last = z, z, z - return - } - - for { - i, ok := t.find(q, k) - if ok { - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i + 1 - p = x - q = x.x[i+1].ch - continue - case *d: - oldV = x.d[i].v - newV, written = upd(oldV, true) - if !written { - return - } - - x.d[i].v = newV - } - return - } - - switch x := q.(type) { - case *x: - if x.c > 2*kx { - x, i = t.splitX(p, x, pi, i) - } - pi = i - p = x - q = x.x[i].ch - case *d: // new KV pair - newV, written = upd(newV, false) - if !written { - return - } - - switch { - case x.c < 2*kd: - t.insert(x, i, k, newV) - default: - t.overflow(p, x, pi, i, k, newV) - } - return - } - } -} - -func (t *Tree) split(p *x, q *d, pi, i int, k int, v int) { - t.ver++ - r := btDPool.Get().(*d) - if q.n != nil { - r.n = q.n - r.n.p = r - } else { - t.last = r - } - q.n = r - r.p = q - - copy(r.d[:], q.d[kd:2*kd]) - for i := range q.d[kd:] { - q.d[kd+i] = zde - } - q.c = kd - r.c = kd - var done bool - if i > kd { - done = true - t.insert(r, i-kd, k, v) - } - if pi >= 0 { - p.insert(pi, r.d[0].k, r) - } else { - t.r = newX(q).insert(0, r.d[0].k, r) - } - if done { - return - } - - t.insert(q, i, k, v) -} - -func (t *Tree) splitX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - r := btXPool.Get().(*x) - copy(r.x[:], q.x[kx+1:]) - q.c = kx - r.c = kx - if pi >= 0 { - p.insert(pi, q.x[kx].k, r) - q.x[kx].k = zk - for i := range q.x[kx+1:] { - q.x[kx+i+1] = zxe - } - - switch { - case i < kx: - return q, i - case i == kx: - return p, pi - default: // i > kx - return r, i - kx - 1 - } - } - - nr := newX(q).insert(0, q.x[kx].k, r) - t.r = nr - q.x[kx].k = zk - for i := range q.x[kx+1:] { - q.x[kx+i+1] = zxe - } - - switch { - case i < kx: - return q, i - case i == kx: - return nr, 0 - default: // i > kx - return r, i - kx - 1 - } -} - -func (t *Tree) underflow(p *x, q *d, pi int) { - t.ver++ - l, r := p.siblings(pi) - - if l != nil && l.c+q.c >= 2*kd { - l.mvR(q, 1) - p.x[pi-1].k = q.d[0].k - return - } - - if r != nil && q.c+r.c >= 2*kd { - q.mvL(r, 1) - p.x[pi].k = r.d[0].k - r.d[r.c] = zde // GC - return - } - - if l != nil { - t.cat(p, l, q, pi-1) - return - } - - t.cat(p, q, r, pi) -} - -func (t *Tree) underflowX(p *x, q *x, pi int, i int) (*x, int) { - t.ver++ - var l, r *x - - if pi >= 0 { - if pi > 0 { - l = p.x[pi-1].ch.(*x) - } - if pi < p.c { - r = p.x[pi+1].ch.(*x) - } - } - - if l != nil && l.c > kx { - q.x[q.c+1].ch = q.x[q.c].ch - copy(q.x[1:], q.x[:q.c]) - q.x[0].ch = l.x[l.c].ch - q.x[0].k = p.x[pi-1].k - q.c++ - i++ - l.c-- - p.x[pi-1].k = l.x[l.c].k - return q, i - } - - if r != nil && r.c > kx { - q.x[q.c].k = p.x[pi].k - q.c++ - q.x[q.c].ch = r.x[0].ch - p.x[pi].k = r.x[0].k - copy(r.x[:], r.x[1:r.c]) - r.c-- - rc := r.c - r.x[rc].ch = r.x[rc+1].ch - r.x[rc].k = zk - r.x[rc+1].ch = nil - return q, i - } - - if l != nil { - i += l.c + 1 - t.catX(p, l, q, pi-1) - q = l - return q, i - } - - t.catX(p, q, r, pi) - return q, i -} - -// ----------------------------------------------------------------- Enumerator - -// Close recycles e to a pool for possible later reuse. No references to e -// should exist or such references must not be used afterwards. -func (e *Enumerator) Close() { - *e = ze - btEPool.Put(e) -} - -// Next returns the currently enumerated item, if it exists and moves to the -// next item in the key collation order. If there is no item to return, err == -// io.EOF is returned. -func (e *Enumerator) Next() (k int, v int, err error) { - if err = e.err; err != nil { - return - } - - if e.ver != e.t.ver { - f, hit := e.t.Seek(e.k) - if !e.hit && hit { - if err = f.next(); err != nil { - return - } - } - - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return - } - - if e.i >= e.q.c { - if err = e.next(); err != nil { - return - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, false - e.next() - return -} - -func (e *Enumerator) next() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i < e.q.c-1: - e.i++ - default: - if e.q, e.i = e.q.n, 0; e.q == nil { - e.err = io.EOF - } - } - return e.err -} - -// Prev returns the currently enumerated item, if it exists and moves to the -// previous item in the key collation order. If there is no item to return, err -// == io.EOF is returned. -func (e *Enumerator) Prev() (k int, v int, err error) { - if err = e.err; err != nil { - return - } - - if e.ver != e.t.ver { - f, hit := e.t.Seek(e.k) - if !e.hit && hit { - if err = f.prev(); err != nil { - return - } - } - - *e = *f - f.Close() - } - if e.q == nil { - e.err, err = io.EOF, io.EOF - return - } - - if e.i >= e.q.c { - if err = e.next(); err != nil { - return - } - } - - i := e.q.d[e.i] - k, v = i.k, i.v - e.k, e.hit = k, false - e.prev() - return -} - -func (e *Enumerator) prev() error { - if e.q == nil { - e.err = io.EOF - return io.EOF - } - - switch { - case e.i > 0: - e.i-- - default: - if e.q = e.q.p; e.q == nil { - e.err = io.EOF - break - } - - e.i = e.q.c - 1 - } - return e.err -} diff --git a/vendor/github.com/davecgh/go-spew/LICENSE b/vendor/github.com/davecgh/go-spew/LICENSE new file mode 100644 index 00000000..c8364161 --- /dev/null +++ b/vendor/github.com/davecgh/go-spew/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/vendor/github.com/davecgh/go-spew/spew/bypass.go b/vendor/github.com/davecgh/go-spew/spew/bypass.go index 565bf589..8a4a6589 100644 --- a/vendor/github.com/davecgh/go-spew/spew/bypass.go +++ b/vendor/github.com/davecgh/go-spew/spew/bypass.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015 Dave Collins +// Copyright (c) 2015-2016 Dave Collins // // Permission to use, copy, modify, and distribute this software for any // purpose with or without fee is hereby granted, provided that the above @@ -13,9 +13,10 @@ // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // NOTE: Due to the following build constraints, this file will only be compiled -// when the code is not running on Google App Engine and "-tags disableunsafe" -// is not added to the go build command line. -// +build !appengine,!disableunsafe +// when the code is not running on Google App Engine, compiled by GopherJS, and +// "-tags safe" is not added to the go build command line. The "disableunsafe" +// tag is deprecated and thus should not be used. +// +build !js,!appengine,!safe,!disableunsafe package spew diff --git a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go index 457e4123..1fe3cf3d 100644 --- a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go +++ b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015 Dave Collins +// Copyright (c) 2015-2016 Dave Collins // // Permission to use, copy, modify, and distribute this software for any // purpose with or without fee is hereby granted, provided that the above @@ -13,9 +13,10 @@ // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. // NOTE: Due to the following build constraints, this file will only be compiled -// when either the code is running on Google App Engine or "-tags disableunsafe" -// is added to the go build command line. -// +build appengine disableunsafe +// when the code is running on Google App Engine, compiled by GopherJS, or +// "-tags safe" is added to the go build command line. The "disableunsafe" +// tag is deprecated and thus should not be used. +// +build js appengine safe disableunsafe package spew diff --git a/vendor/github.com/davecgh/go-spew/spew/common.go b/vendor/github.com/davecgh/go-spew/spew/common.go index 14f02dc1..7c519ff4 100644 --- a/vendor/github.com/davecgh/go-spew/spew/common.go +++ b/vendor/github.com/davecgh/go-spew/spew/common.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Dave Collins + * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/vendor/github.com/davecgh/go-spew/spew/common_test.go b/vendor/github.com/davecgh/go-spew/spew/common_test.go deleted file mode 100644 index 39b7525b..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/common_test.go +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Copyright (c) 2013 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew_test - -import ( - "fmt" - "reflect" - "testing" - - "github.com/davecgh/go-spew/spew" -) - -// custom type to test Stinger interface on non-pointer receiver. -type stringer string - -// String implements the Stringer interface for testing invocation of custom -// stringers on types with non-pointer receivers. -func (s stringer) String() string { - return "stringer " + string(s) -} - -// custom type to test Stinger interface on pointer receiver. -type pstringer string - -// String implements the Stringer interface for testing invocation of custom -// stringers on types with only pointer receivers. -func (s *pstringer) String() string { - return "stringer " + string(*s) -} - -// xref1 and xref2 are cross referencing structs for testing circular reference -// detection. -type xref1 struct { - ps2 *xref2 -} -type xref2 struct { - ps1 *xref1 -} - -// indirCir1, indirCir2, and indirCir3 are used to generate an indirect circular -// reference for testing detection. -type indirCir1 struct { - ps2 *indirCir2 -} -type indirCir2 struct { - ps3 *indirCir3 -} -type indirCir3 struct { - ps1 *indirCir1 -} - -// embed is used to test embedded structures. -type embed struct { - a string -} - -// embedwrap is used to test embedded structures. -type embedwrap struct { - *embed - e *embed -} - -// panicer is used to intentionally cause a panic for testing spew properly -// handles them -type panicer int - -func (p panicer) String() string { - panic("test panic") -} - -// customError is used to test custom error interface invocation. -type customError int - -func (e customError) Error() string { - return fmt.Sprintf("error: %d", int(e)) -} - -// stringizeWants converts a slice of wanted test output into a format suitable -// for a test error message. -func stringizeWants(wants []string) string { - s := "" - for i, want := range wants { - if i > 0 { - s += fmt.Sprintf("want%d: %s", i+1, want) - } else { - s += "want: " + want - } - } - return s -} - -// testFailed returns whether or not a test failed by checking if the result -// of the test is in the slice of wanted strings. -func testFailed(result string, wants []string) bool { - for _, want := range wants { - if result == want { - return false - } - } - return true -} - -type sortableStruct struct { - x int -} - -func (ss sortableStruct) String() string { - return fmt.Sprintf("ss.%d", ss.x) -} - -type unsortableStruct struct { - x int -} - -type sortTestCase struct { - input []reflect.Value - expected []reflect.Value -} - -func helpTestSortValues(tests []sortTestCase, cs *spew.ConfigState, t *testing.T) { - getInterfaces := func(values []reflect.Value) []interface{} { - interfaces := []interface{}{} - for _, v := range values { - interfaces = append(interfaces, v.Interface()) - } - return interfaces - } - - for _, test := range tests { - spew.SortValues(test.input, cs) - // reflect.DeepEqual cannot really make sense of reflect.Value, - // probably because of all the pointer tricks. For instance, - // v(2.0) != v(2.0) on a 32-bits system. Turn them into interface{} - // instead. - input := getInterfaces(test.input) - expected := getInterfaces(test.expected) - if !reflect.DeepEqual(input, expected) { - t.Errorf("Sort mismatch:\n %v != %v", input, expected) - } - } -} - -// TestSortValues ensures the sort functionality for relect.Value based sorting -// works as intended. -func TestSortValues(t *testing.T) { - v := reflect.ValueOf - - a := v("a") - b := v("b") - c := v("c") - embedA := v(embed{"a"}) - embedB := v(embed{"b"}) - embedC := v(embed{"c"}) - tests := []sortTestCase{ - // No values. - { - []reflect.Value{}, - []reflect.Value{}, - }, - // Bools. - { - []reflect.Value{v(false), v(true), v(false)}, - []reflect.Value{v(false), v(false), v(true)}, - }, - // Ints. - { - []reflect.Value{v(2), v(1), v(3)}, - []reflect.Value{v(1), v(2), v(3)}, - }, - // Uints. - { - []reflect.Value{v(uint8(2)), v(uint8(1)), v(uint8(3))}, - []reflect.Value{v(uint8(1)), v(uint8(2)), v(uint8(3))}, - }, - // Floats. - { - []reflect.Value{v(2.0), v(1.0), v(3.0)}, - []reflect.Value{v(1.0), v(2.0), v(3.0)}, - }, - // Strings. - { - []reflect.Value{b, a, c}, - []reflect.Value{a, b, c}, - }, - // Array - { - []reflect.Value{v([3]int{3, 2, 1}), v([3]int{1, 3, 2}), v([3]int{1, 2, 3})}, - []reflect.Value{v([3]int{1, 2, 3}), v([3]int{1, 3, 2}), v([3]int{3, 2, 1})}, - }, - // Uintptrs. - { - []reflect.Value{v(uintptr(2)), v(uintptr(1)), v(uintptr(3))}, - []reflect.Value{v(uintptr(1)), v(uintptr(2)), v(uintptr(3))}, - }, - // SortableStructs. - { - // Note: not sorted - DisableMethods is set. - []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, - []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, - }, - // UnsortableStructs. - { - // Note: not sorted - SpewKeys is false. - []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, - []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, - }, - // Invalid. - { - []reflect.Value{embedB, embedA, embedC}, - []reflect.Value{embedB, embedA, embedC}, - }, - } - cs := spew.ConfigState{DisableMethods: true, SpewKeys: false} - helpTestSortValues(tests, &cs, t) -} - -// TestSortValuesWithMethods ensures the sort functionality for relect.Value -// based sorting works as intended when using string methods. -func TestSortValuesWithMethods(t *testing.T) { - v := reflect.ValueOf - - a := v("a") - b := v("b") - c := v("c") - tests := []sortTestCase{ - // Ints. - { - []reflect.Value{v(2), v(1), v(3)}, - []reflect.Value{v(1), v(2), v(3)}, - }, - // Strings. - { - []reflect.Value{b, a, c}, - []reflect.Value{a, b, c}, - }, - // SortableStructs. - { - []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, - []reflect.Value{v(sortableStruct{1}), v(sortableStruct{2}), v(sortableStruct{3})}, - }, - // UnsortableStructs. - { - // Note: not sorted - SpewKeys is false. - []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, - []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, - }, - } - cs := spew.ConfigState{DisableMethods: false, SpewKeys: false} - helpTestSortValues(tests, &cs, t) -} - -// TestSortValuesWithSpew ensures the sort functionality for relect.Value -// based sorting works as intended when using spew to stringify keys. -func TestSortValuesWithSpew(t *testing.T) { - v := reflect.ValueOf - - a := v("a") - b := v("b") - c := v("c") - tests := []sortTestCase{ - // Ints. - { - []reflect.Value{v(2), v(1), v(3)}, - []reflect.Value{v(1), v(2), v(3)}, - }, - // Strings. - { - []reflect.Value{b, a, c}, - []reflect.Value{a, b, c}, - }, - // SortableStructs. - { - []reflect.Value{v(sortableStruct{2}), v(sortableStruct{1}), v(sortableStruct{3})}, - []reflect.Value{v(sortableStruct{1}), v(sortableStruct{2}), v(sortableStruct{3})}, - }, - // UnsortableStructs. - { - []reflect.Value{v(unsortableStruct{2}), v(unsortableStruct{1}), v(unsortableStruct{3})}, - []reflect.Value{v(unsortableStruct{1}), v(unsortableStruct{2}), v(unsortableStruct{3})}, - }, - } - cs := spew.ConfigState{DisableMethods: true, SpewKeys: true} - helpTestSortValues(tests, &cs, t) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/config.go b/vendor/github.com/davecgh/go-spew/spew/config.go index ee1ab07b..2e3d22f3 100644 --- a/vendor/github.com/davecgh/go-spew/spew/config.go +++ b/vendor/github.com/davecgh/go-spew/spew/config.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Dave Collins + * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -64,9 +64,18 @@ type ConfigState struct { // inside these interface methods. As a result, this option relies on // access to the unsafe package, so it will not have any effect when // running in environments without access to the unsafe package such as - // Google App Engine or with the "disableunsafe" build tag specified. + // Google App Engine or with the "safe" build tag specified. DisablePointerMethods bool + // DisablePointerAddresses specifies whether to disable the printing of + // pointer addresses. This is useful when diffing data structures in tests. + DisablePointerAddresses bool + + // DisableCapacities specifies whether to disable the printing of capacities + // for arrays, slices, maps and channels. This is useful when diffing + // data structures in tests. + DisableCapacities bool + // ContinueOnMethod specifies whether or not recursion should continue once // a custom error or Stringer interface is invoked. The default, false, // means it will print the results of invoking the custom error or Stringer diff --git a/vendor/github.com/davecgh/go-spew/spew/doc.go b/vendor/github.com/davecgh/go-spew/spew/doc.go index 5be0c406..aacaac6f 100644 --- a/vendor/github.com/davecgh/go-spew/spew/doc.go +++ b/vendor/github.com/davecgh/go-spew/spew/doc.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Dave Collins + * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -91,6 +91,15 @@ The following configuration options are available: which only accept pointer receivers from non-pointer variables. Pointer method invocation is enabled by default. + * DisablePointerAddresses + DisablePointerAddresses specifies whether to disable the printing of + pointer addresses. This is useful when diffing data structures in tests. + + * DisableCapacities + DisableCapacities specifies whether to disable the printing of + capacities for arrays, slices, maps and channels. This is useful when + diffing data structures in tests. + * ContinueOnMethod Enables recursion into types after invoking error and Stringer interface methods. Recursion after method invocation is disabled by default. diff --git a/vendor/github.com/davecgh/go-spew/spew/dump.go b/vendor/github.com/davecgh/go-spew/spew/dump.go index a0ff95e2..df1d582a 100644 --- a/vendor/github.com/davecgh/go-spew/spew/dump.go +++ b/vendor/github.com/davecgh/go-spew/spew/dump.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Dave Collins + * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -129,7 +129,7 @@ func (d *dumpState) dumpPtr(v reflect.Value) { d.w.Write(closeParenBytes) // Display pointer information. - if len(pointerChain) > 0 { + if !d.cs.DisablePointerAddresses && len(pointerChain) > 0 { d.w.Write(openParenBytes) for i, addr := range pointerChain { if i > 0 { @@ -282,13 +282,13 @@ func (d *dumpState) dump(v reflect.Value) { case reflect.Map, reflect.String: valueLen = v.Len() } - if valueLen != 0 || valueCap != 0 { + if valueLen != 0 || !d.cs.DisableCapacities && valueCap != 0 { d.w.Write(openParenBytes) if valueLen != 0 { d.w.Write(lenEqualsBytes) printInt(d.w, int64(valueLen), 10) } - if valueCap != 0 { + if !d.cs.DisableCapacities && valueCap != 0 { if valueLen != 0 { d.w.Write(spaceBytes) } diff --git a/vendor/github.com/davecgh/go-spew/spew/dump_test.go b/vendor/github.com/davecgh/go-spew/spew/dump_test.go deleted file mode 100644 index 2b320401..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/dump_test.go +++ /dev/null @@ -1,1042 +0,0 @@ -/* - * Copyright (c) 2013 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* -Test Summary: -NOTE: For each test, a nil pointer, a single pointer and double pointer to the -base test element are also tested to ensure proper indirection across all types. - -- Max int8, int16, int32, int64, int -- Max uint8, uint16, uint32, uint64, uint -- Boolean true and false -- Standard complex64 and complex128 -- Array containing standard ints -- Array containing type with custom formatter on pointer receiver only -- Array containing interfaces -- Array containing bytes -- Slice containing standard float32 values -- Slice containing type with custom formatter on pointer receiver only -- Slice containing interfaces -- Slice containing bytes -- Nil slice -- Standard string -- Nil interface -- Sub-interface -- Map with string keys and int vals -- Map with custom formatter type on pointer receiver only keys and vals -- Map with interface keys and values -- Map with nil interface value -- Struct with primitives -- Struct that contains another struct -- Struct that contains custom type with Stringer pointer interface via both - exported and unexported fields -- Struct that contains embedded struct and field to same struct -- Uintptr to 0 (null pointer) -- Uintptr address of real variable -- Unsafe.Pointer to 0 (null pointer) -- Unsafe.Pointer to address of real variable -- Nil channel -- Standard int channel -- Function with no params and no returns -- Function with param and no returns -- Function with multiple params and multiple returns -- Struct that is circular through self referencing -- Structs that are circular through cross referencing -- Structs that are indirectly circular -- Type that panics in its Stringer interface -*/ - -package spew_test - -import ( - "bytes" - "fmt" - "testing" - "unsafe" - - "github.com/davecgh/go-spew/spew" -) - -// dumpTest is used to describe a test to be perfomed against the Dump method. -type dumpTest struct { - in interface{} - wants []string -} - -// dumpTests houses all of the tests to be performed against the Dump method. -var dumpTests = make([]dumpTest, 0) - -// addDumpTest is a helper method to append the passed input and desired result -// to dumpTests -func addDumpTest(in interface{}, wants ...string) { - test := dumpTest{in, wants} - dumpTests = append(dumpTests, test) -} - -func addIntDumpTests() { - // Max int8. - v := int8(127) - nv := (*int8)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "int8" - vs := "127" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Max int16. - v2 := int16(32767) - nv2 := (*int16)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "int16" - v2s := "32767" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*"+v2t+")()\n") - - // Max int32. - v3 := int32(2147483647) - nv3 := (*int32)(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "int32" - v3s := "2147483647" - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") - addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") - addDumpTest(nv3, "(*"+v3t+")()\n") - - // Max int64. - v4 := int64(9223372036854775807) - nv4 := (*int64)(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "int64" - v4s := "9223372036854775807" - addDumpTest(v4, "("+v4t+") "+v4s+"\n") - addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") - addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") - addDumpTest(nv4, "(*"+v4t+")()\n") - - // Max int. - v5 := int(2147483647) - nv5 := (*int)(nil) - pv5 := &v5 - v5Addr := fmt.Sprintf("%p", pv5) - pv5Addr := fmt.Sprintf("%p", &pv5) - v5t := "int" - v5s := "2147483647" - addDumpTest(v5, "("+v5t+") "+v5s+"\n") - addDumpTest(pv5, "(*"+v5t+")("+v5Addr+")("+v5s+")\n") - addDumpTest(&pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")("+v5s+")\n") - addDumpTest(nv5, "(*"+v5t+")()\n") -} - -func addUintDumpTests() { - // Max uint8. - v := uint8(255) - nv := (*uint8)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "uint8" - vs := "255" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Max uint16. - v2 := uint16(65535) - nv2 := (*uint16)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "uint16" - v2s := "65535" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*"+v2t+")()\n") - - // Max uint32. - v3 := uint32(4294967295) - nv3 := (*uint32)(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "uint32" - v3s := "4294967295" - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") - addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") - addDumpTest(nv3, "(*"+v3t+")()\n") - - // Max uint64. - v4 := uint64(18446744073709551615) - nv4 := (*uint64)(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "uint64" - v4s := "18446744073709551615" - addDumpTest(v4, "("+v4t+") "+v4s+"\n") - addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") - addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") - addDumpTest(nv4, "(*"+v4t+")()\n") - - // Max uint. - v5 := uint(4294967295) - nv5 := (*uint)(nil) - pv5 := &v5 - v5Addr := fmt.Sprintf("%p", pv5) - pv5Addr := fmt.Sprintf("%p", &pv5) - v5t := "uint" - v5s := "4294967295" - addDumpTest(v5, "("+v5t+") "+v5s+"\n") - addDumpTest(pv5, "(*"+v5t+")("+v5Addr+")("+v5s+")\n") - addDumpTest(&pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")("+v5s+")\n") - addDumpTest(nv5, "(*"+v5t+")()\n") -} - -func addBoolDumpTests() { - // Boolean true. - v := bool(true) - nv := (*bool)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "bool" - vs := "true" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Boolean false. - v2 := bool(false) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "bool" - v2s := "false" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") -} - -func addFloatDumpTests() { - // Standard float32. - v := float32(3.1415) - nv := (*float32)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "float32" - vs := "3.1415" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Standard float64. - v2 := float64(3.1415926) - nv2 := (*float64)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "float64" - v2s := "3.1415926" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*"+v2t+")()\n") -} - -func addComplexDumpTests() { - // Standard complex64. - v := complex(float32(6), -2) - nv := (*complex64)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "complex64" - vs := "(6-2i)" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Standard complex128. - v2 := complex(float64(-6), 2) - nv2 := (*complex128)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "complex128" - v2s := "(-6+2i)" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*"+v2t+")()\n") -} - -func addArrayDumpTests() { - // Array containing standard ints. - v := [3]int{1, 2, 3} - vLen := fmt.Sprintf("%d", len(v)) - vCap := fmt.Sprintf("%d", cap(v)) - nv := (*[3]int)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "int" - vs := "(len=" + vLen + " cap=" + vCap + ") {\n (" + vt + ") 1,\n (" + - vt + ") 2,\n (" + vt + ") 3\n}" - addDumpTest(v, "([3]"+vt+") "+vs+"\n") - addDumpTest(pv, "(*[3]"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**[3]"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*[3]"+vt+")()\n") - - // Array containing type with custom formatter on pointer receiver only. - v2i0 := pstringer("1") - v2i1 := pstringer("2") - v2i2 := pstringer("3") - v2 := [3]pstringer{v2i0, v2i1, v2i2} - v2i0Len := fmt.Sprintf("%d", len(v2i0)) - v2i1Len := fmt.Sprintf("%d", len(v2i1)) - v2i2Len := fmt.Sprintf("%d", len(v2i2)) - v2Len := fmt.Sprintf("%d", len(v2)) - v2Cap := fmt.Sprintf("%d", cap(v2)) - nv2 := (*[3]pstringer)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "spew_test.pstringer" - v2sp := "(len=" + v2Len + " cap=" + v2Cap + ") {\n (" + v2t + - ") (len=" + v2i0Len + ") stringer 1,\n (" + v2t + - ") (len=" + v2i1Len + ") stringer 2,\n (" + v2t + - ") (len=" + v2i2Len + ") " + "stringer 3\n}" - v2s := v2sp - if spew.UnsafeDisabled { - v2s = "(len=" + v2Len + " cap=" + v2Cap + ") {\n (" + v2t + - ") (len=" + v2i0Len + ") \"1\",\n (" + v2t + ") (len=" + - v2i1Len + ") \"2\",\n (" + v2t + ") (len=" + v2i2Len + - ") " + "\"3\"\n}" - } - addDumpTest(v2, "([3]"+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*[3]"+v2t+")("+v2Addr+")("+v2sp+")\n") - addDumpTest(&pv2, "(**[3]"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2sp+")\n") - addDumpTest(nv2, "(*[3]"+v2t+")()\n") - - // Array containing interfaces. - v3i0 := "one" - v3 := [3]interface{}{v3i0, int(2), uint(3)} - v3i0Len := fmt.Sprintf("%d", len(v3i0)) - v3Len := fmt.Sprintf("%d", len(v3)) - v3Cap := fmt.Sprintf("%d", cap(v3)) - nv3 := (*[3]interface{})(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "[3]interface {}" - v3t2 := "string" - v3t3 := "int" - v3t4 := "uint" - v3s := "(len=" + v3Len + " cap=" + v3Cap + ") {\n (" + v3t2 + ") " + - "(len=" + v3i0Len + ") \"one\",\n (" + v3t3 + ") 2,\n (" + - v3t4 + ") 3\n}" - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") - addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") - addDumpTest(nv3, "(*"+v3t+")()\n") - - // Array containing bytes. - v4 := [34]byte{ - 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, - 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, - 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, - 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, - 0x31, 0x32, - } - v4Len := fmt.Sprintf("%d", len(v4)) - v4Cap := fmt.Sprintf("%d", cap(v4)) - nv4 := (*[34]byte)(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "[34]uint8" - v4s := "(len=" + v4Len + " cap=" + v4Cap + ") " + - "{\n 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20" + - " |............... |\n" + - " 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30" + - " |!\"#$%&'()*+,-./0|\n" + - " 00000020 31 32 " + - " |12|\n}" - addDumpTest(v4, "("+v4t+") "+v4s+"\n") - addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") - addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") - addDumpTest(nv4, "(*"+v4t+")()\n") -} - -func addSliceDumpTests() { - // Slice containing standard float32 values. - v := []float32{3.14, 6.28, 12.56} - vLen := fmt.Sprintf("%d", len(v)) - vCap := fmt.Sprintf("%d", cap(v)) - nv := (*[]float32)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "float32" - vs := "(len=" + vLen + " cap=" + vCap + ") {\n (" + vt + ") 3.14,\n (" + - vt + ") 6.28,\n (" + vt + ") 12.56\n}" - addDumpTest(v, "([]"+vt+") "+vs+"\n") - addDumpTest(pv, "(*[]"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**[]"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*[]"+vt+")()\n") - - // Slice containing type with custom formatter on pointer receiver only. - v2i0 := pstringer("1") - v2i1 := pstringer("2") - v2i2 := pstringer("3") - v2 := []pstringer{v2i0, v2i1, v2i2} - v2i0Len := fmt.Sprintf("%d", len(v2i0)) - v2i1Len := fmt.Sprintf("%d", len(v2i1)) - v2i2Len := fmt.Sprintf("%d", len(v2i2)) - v2Len := fmt.Sprintf("%d", len(v2)) - v2Cap := fmt.Sprintf("%d", cap(v2)) - nv2 := (*[]pstringer)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "spew_test.pstringer" - v2s := "(len=" + v2Len + " cap=" + v2Cap + ") {\n (" + v2t + ") (len=" + - v2i0Len + ") stringer 1,\n (" + v2t + ") (len=" + v2i1Len + - ") stringer 2,\n (" + v2t + ") (len=" + v2i2Len + ") " + - "stringer 3\n}" - addDumpTest(v2, "([]"+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*[]"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**[]"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*[]"+v2t+")()\n") - - // Slice containing interfaces. - v3i0 := "one" - v3 := []interface{}{v3i0, int(2), uint(3), nil} - v3i0Len := fmt.Sprintf("%d", len(v3i0)) - v3Len := fmt.Sprintf("%d", len(v3)) - v3Cap := fmt.Sprintf("%d", cap(v3)) - nv3 := (*[]interface{})(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "[]interface {}" - v3t2 := "string" - v3t3 := "int" - v3t4 := "uint" - v3t5 := "interface {}" - v3s := "(len=" + v3Len + " cap=" + v3Cap + ") {\n (" + v3t2 + ") " + - "(len=" + v3i0Len + ") \"one\",\n (" + v3t3 + ") 2,\n (" + - v3t4 + ") 3,\n (" + v3t5 + ") \n}" - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") - addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") - addDumpTest(nv3, "(*"+v3t+")()\n") - - // Slice containing bytes. - v4 := []byte{ - 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, - 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, - 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, - 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, - 0x31, 0x32, - } - v4Len := fmt.Sprintf("%d", len(v4)) - v4Cap := fmt.Sprintf("%d", cap(v4)) - nv4 := (*[]byte)(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "[]uint8" - v4s := "(len=" + v4Len + " cap=" + v4Cap + ") " + - "{\n 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20" + - " |............... |\n" + - " 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30" + - " |!\"#$%&'()*+,-./0|\n" + - " 00000020 31 32 " + - " |12|\n}" - addDumpTest(v4, "("+v4t+") "+v4s+"\n") - addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") - addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") - addDumpTest(nv4, "(*"+v4t+")()\n") - - // Nil slice. - v5 := []int(nil) - nv5 := (*[]int)(nil) - pv5 := &v5 - v5Addr := fmt.Sprintf("%p", pv5) - pv5Addr := fmt.Sprintf("%p", &pv5) - v5t := "[]int" - v5s := "" - addDumpTest(v5, "("+v5t+") "+v5s+"\n") - addDumpTest(pv5, "(*"+v5t+")("+v5Addr+")("+v5s+")\n") - addDumpTest(&pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")("+v5s+")\n") - addDumpTest(nv5, "(*"+v5t+")()\n") -} - -func addStringDumpTests() { - // Standard string. - v := "test" - vLen := fmt.Sprintf("%d", len(v)) - nv := (*string)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "string" - vs := "(len=" + vLen + ") \"test\"" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") -} - -func addInterfaceDumpTests() { - // Nil interface. - var v interface{} - nv := (*interface{})(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "interface {}" - vs := "" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Sub-interface. - v2 := interface{}(uint16(65535)) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "uint16" - v2s := "65535" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") -} - -func addMapDumpTests() { - // Map with string keys and int vals. - k := "one" - kk := "two" - m := map[string]int{k: 1, kk: 2} - klen := fmt.Sprintf("%d", len(k)) // not kLen to shut golint up - kkLen := fmt.Sprintf("%d", len(kk)) - mLen := fmt.Sprintf("%d", len(m)) - nilMap := map[string]int(nil) - nm := (*map[string]int)(nil) - pm := &m - mAddr := fmt.Sprintf("%p", pm) - pmAddr := fmt.Sprintf("%p", &pm) - mt := "map[string]int" - mt1 := "string" - mt2 := "int" - ms := "(len=" + mLen + ") {\n (" + mt1 + ") (len=" + klen + ") " + - "\"one\": (" + mt2 + ") 1,\n (" + mt1 + ") (len=" + kkLen + - ") \"two\": (" + mt2 + ") 2\n}" - ms2 := "(len=" + mLen + ") {\n (" + mt1 + ") (len=" + kkLen + ") " + - "\"two\": (" + mt2 + ") 2,\n (" + mt1 + ") (len=" + klen + - ") \"one\": (" + mt2 + ") 1\n}" - addDumpTest(m, "("+mt+") "+ms+"\n", "("+mt+") "+ms2+"\n") - addDumpTest(pm, "(*"+mt+")("+mAddr+")("+ms+")\n", - "(*"+mt+")("+mAddr+")("+ms2+")\n") - addDumpTest(&pm, "(**"+mt+")("+pmAddr+"->"+mAddr+")("+ms+")\n", - "(**"+mt+")("+pmAddr+"->"+mAddr+")("+ms2+")\n") - addDumpTest(nm, "(*"+mt+")()\n") - addDumpTest(nilMap, "("+mt+") \n") - - // Map with custom formatter type on pointer receiver only keys and vals. - k2 := pstringer("one") - v2 := pstringer("1") - m2 := map[pstringer]pstringer{k2: v2} - k2Len := fmt.Sprintf("%d", len(k2)) - v2Len := fmt.Sprintf("%d", len(v2)) - m2Len := fmt.Sprintf("%d", len(m2)) - nilMap2 := map[pstringer]pstringer(nil) - nm2 := (*map[pstringer]pstringer)(nil) - pm2 := &m2 - m2Addr := fmt.Sprintf("%p", pm2) - pm2Addr := fmt.Sprintf("%p", &pm2) - m2t := "map[spew_test.pstringer]spew_test.pstringer" - m2t1 := "spew_test.pstringer" - m2t2 := "spew_test.pstringer" - m2s := "(len=" + m2Len + ") {\n (" + m2t1 + ") (len=" + k2Len + ") " + - "stringer one: (" + m2t2 + ") (len=" + v2Len + ") stringer 1\n}" - if spew.UnsafeDisabled { - m2s = "(len=" + m2Len + ") {\n (" + m2t1 + ") (len=" + k2Len + - ") " + "\"one\": (" + m2t2 + ") (len=" + v2Len + - ") \"1\"\n}" - } - addDumpTest(m2, "("+m2t+") "+m2s+"\n") - addDumpTest(pm2, "(*"+m2t+")("+m2Addr+")("+m2s+")\n") - addDumpTest(&pm2, "(**"+m2t+")("+pm2Addr+"->"+m2Addr+")("+m2s+")\n") - addDumpTest(nm2, "(*"+m2t+")()\n") - addDumpTest(nilMap2, "("+m2t+") \n") - - // Map with interface keys and values. - k3 := "one" - k3Len := fmt.Sprintf("%d", len(k3)) - m3 := map[interface{}]interface{}{k3: 1} - m3Len := fmt.Sprintf("%d", len(m3)) - nilMap3 := map[interface{}]interface{}(nil) - nm3 := (*map[interface{}]interface{})(nil) - pm3 := &m3 - m3Addr := fmt.Sprintf("%p", pm3) - pm3Addr := fmt.Sprintf("%p", &pm3) - m3t := "map[interface {}]interface {}" - m3t1 := "string" - m3t2 := "int" - m3s := "(len=" + m3Len + ") {\n (" + m3t1 + ") (len=" + k3Len + ") " + - "\"one\": (" + m3t2 + ") 1\n}" - addDumpTest(m3, "("+m3t+") "+m3s+"\n") - addDumpTest(pm3, "(*"+m3t+")("+m3Addr+")("+m3s+")\n") - addDumpTest(&pm3, "(**"+m3t+")("+pm3Addr+"->"+m3Addr+")("+m3s+")\n") - addDumpTest(nm3, "(*"+m3t+")()\n") - addDumpTest(nilMap3, "("+m3t+") \n") - - // Map with nil interface value. - k4 := "nil" - k4Len := fmt.Sprintf("%d", len(k4)) - m4 := map[string]interface{}{k4: nil} - m4Len := fmt.Sprintf("%d", len(m4)) - nilMap4 := map[string]interface{}(nil) - nm4 := (*map[string]interface{})(nil) - pm4 := &m4 - m4Addr := fmt.Sprintf("%p", pm4) - pm4Addr := fmt.Sprintf("%p", &pm4) - m4t := "map[string]interface {}" - m4t1 := "string" - m4t2 := "interface {}" - m4s := "(len=" + m4Len + ") {\n (" + m4t1 + ") (len=" + k4Len + ")" + - " \"nil\": (" + m4t2 + ") \n}" - addDumpTest(m4, "("+m4t+") "+m4s+"\n") - addDumpTest(pm4, "(*"+m4t+")("+m4Addr+")("+m4s+")\n") - addDumpTest(&pm4, "(**"+m4t+")("+pm4Addr+"->"+m4Addr+")("+m4s+")\n") - addDumpTest(nm4, "(*"+m4t+")()\n") - addDumpTest(nilMap4, "("+m4t+") \n") -} - -func addStructDumpTests() { - // Struct with primitives. - type s1 struct { - a int8 - b uint8 - } - v := s1{127, 255} - nv := (*s1)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.s1" - vt2 := "int8" - vt3 := "uint8" - vs := "{\n a: (" + vt2 + ") 127,\n b: (" + vt3 + ") 255\n}" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Struct that contains another struct. - type s2 struct { - s1 s1 - b bool - } - v2 := s2{s1{127, 255}, true} - nv2 := (*s2)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "spew_test.s2" - v2t2 := "spew_test.s1" - v2t3 := "int8" - v2t4 := "uint8" - v2t5 := "bool" - v2s := "{\n s1: (" + v2t2 + ") {\n a: (" + v2t3 + ") 127,\n b: (" + - v2t4 + ") 255\n },\n b: (" + v2t5 + ") true\n}" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*"+v2t+")()\n") - - // Struct that contains custom type with Stringer pointer interface via both - // exported and unexported fields. - type s3 struct { - s pstringer - S pstringer - } - v3 := s3{"test", "test2"} - nv3 := (*s3)(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "spew_test.s3" - v3t2 := "spew_test.pstringer" - v3s := "{\n s: (" + v3t2 + ") (len=4) stringer test,\n S: (" + v3t2 + - ") (len=5) stringer test2\n}" - v3sp := v3s - if spew.UnsafeDisabled { - v3s = "{\n s: (" + v3t2 + ") (len=4) \"test\",\n S: (" + - v3t2 + ") (len=5) \"test2\"\n}" - v3sp = "{\n s: (" + v3t2 + ") (len=4) \"test\",\n S: (" + - v3t2 + ") (len=5) stringer test2\n}" - } - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3sp+")\n") - addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3sp+")\n") - addDumpTest(nv3, "(*"+v3t+")()\n") - - // Struct that contains embedded struct and field to same struct. - e := embed{"embedstr"} - eLen := fmt.Sprintf("%d", len("embedstr")) - v4 := embedwrap{embed: &e, e: &e} - nv4 := (*embedwrap)(nil) - pv4 := &v4 - eAddr := fmt.Sprintf("%p", &e) - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "spew_test.embedwrap" - v4t2 := "spew_test.embed" - v4t3 := "string" - v4s := "{\n embed: (*" + v4t2 + ")(" + eAddr + ")({\n a: (" + v4t3 + - ") (len=" + eLen + ") \"embedstr\"\n }),\n e: (*" + v4t2 + - ")(" + eAddr + ")({\n a: (" + v4t3 + ") (len=" + eLen + ")" + - " \"embedstr\"\n })\n}" - addDumpTest(v4, "("+v4t+") "+v4s+"\n") - addDumpTest(pv4, "(*"+v4t+")("+v4Addr+")("+v4s+")\n") - addDumpTest(&pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")("+v4s+")\n") - addDumpTest(nv4, "(*"+v4t+")()\n") -} - -func addUintptrDumpTests() { - // Null pointer. - v := uintptr(0) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "uintptr" - vs := "" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - - // Address of real variable. - i := 1 - v2 := uintptr(unsafe.Pointer(&i)) - nv2 := (*uintptr)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "uintptr" - v2s := fmt.Sprintf("%p", &i) - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*"+v2t+")()\n") -} - -func addUnsafePointerDumpTests() { - // Null pointer. - v := unsafe.Pointer(uintptr(0)) - nv := (*unsafe.Pointer)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "unsafe.Pointer" - vs := "" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Address of real variable. - i := 1 - v2 := unsafe.Pointer(&i) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "unsafe.Pointer" - v2s := fmt.Sprintf("%p", &i) - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv, "(*"+vt+")()\n") -} - -func addChanDumpTests() { - // Nil channel. - var v chan int - pv := &v - nv := (*chan int)(nil) - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "chan int" - vs := "" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Real channel. - v2 := make(chan int) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "chan int" - v2s := fmt.Sprintf("%p", v2) - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") -} - -func addFuncDumpTests() { - // Function with no params and no returns. - v := addIntDumpTests - nv := (*func())(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "func()" - vs := fmt.Sprintf("%p", v) - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") - - // Function with param and no returns. - v2 := TestDump - nv2 := (*func(*testing.T))(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "func(*testing.T)" - v2s := fmt.Sprintf("%p", v2) - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s+")\n") - addDumpTest(nv2, "(*"+v2t+")()\n") - - // Function with multiple params and multiple returns. - var v3 = func(i int, s string) (b bool, err error) { - return true, nil - } - nv3 := (*func(int, string) (bool, error))(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "func(int, string) (bool, error)" - v3s := fmt.Sprintf("%p", v3) - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s+")\n") - addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s+")\n") - addDumpTest(nv3, "(*"+v3t+")()\n") -} - -func addCircularDumpTests() { - // Struct that is circular through self referencing. - type circular struct { - c *circular - } - v := circular{nil} - v.c = &v - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.circular" - vs := "{\n c: (*" + vt + ")(" + vAddr + ")({\n c: (*" + vt + ")(" + - vAddr + ")()\n })\n}" - vs2 := "{\n c: (*" + vt + ")(" + vAddr + ")()\n}" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs2+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs2+")\n") - - // Structs that are circular through cross referencing. - v2 := xref1{nil} - ts2 := xref2{&v2} - v2.ps2 = &ts2 - pv2 := &v2 - ts2Addr := fmt.Sprintf("%p", &ts2) - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "spew_test.xref1" - v2t2 := "spew_test.xref2" - v2s := "{\n ps2: (*" + v2t2 + ")(" + ts2Addr + ")({\n ps1: (*" + v2t + - ")(" + v2Addr + ")({\n ps2: (*" + v2t2 + ")(" + ts2Addr + - ")()\n })\n })\n}" - v2s2 := "{\n ps2: (*" + v2t2 + ")(" + ts2Addr + ")({\n ps1: (*" + v2t + - ")(" + v2Addr + ")()\n })\n}" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - addDumpTest(pv2, "(*"+v2t+")("+v2Addr+")("+v2s2+")\n") - addDumpTest(&pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")("+v2s2+")\n") - - // Structs that are indirectly circular. - v3 := indirCir1{nil} - tic2 := indirCir2{nil} - tic3 := indirCir3{&v3} - tic2.ps3 = &tic3 - v3.ps2 = &tic2 - pv3 := &v3 - tic2Addr := fmt.Sprintf("%p", &tic2) - tic3Addr := fmt.Sprintf("%p", &tic3) - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "spew_test.indirCir1" - v3t2 := "spew_test.indirCir2" - v3t3 := "spew_test.indirCir3" - v3s := "{\n ps2: (*" + v3t2 + ")(" + tic2Addr + ")({\n ps3: (*" + v3t3 + - ")(" + tic3Addr + ")({\n ps1: (*" + v3t + ")(" + v3Addr + - ")({\n ps2: (*" + v3t2 + ")(" + tic2Addr + - ")()\n })\n })\n })\n}" - v3s2 := "{\n ps2: (*" + v3t2 + ")(" + tic2Addr + ")({\n ps3: (*" + v3t3 + - ")(" + tic3Addr + ")({\n ps1: (*" + v3t + ")(" + v3Addr + - ")()\n })\n })\n}" - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - addDumpTest(pv3, "(*"+v3t+")("+v3Addr+")("+v3s2+")\n") - addDumpTest(&pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")("+v3s2+")\n") -} - -func addPanicDumpTests() { - // Type that panics in its Stringer interface. - v := panicer(127) - nv := (*panicer)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.panicer" - vs := "(PANIC=test panic)127" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") -} - -func addErrorDumpTests() { - // Type that has a custom Error interface. - v := customError(127) - nv := (*customError)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.customError" - vs := "error: 127" - addDumpTest(v, "("+vt+") "+vs+"\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")("+vs+")\n") - addDumpTest(nv, "(*"+vt+")()\n") -} - -// TestDump executes all of the tests described by dumpTests. -func TestDump(t *testing.T) { - // Setup tests. - addIntDumpTests() - addUintDumpTests() - addBoolDumpTests() - addFloatDumpTests() - addComplexDumpTests() - addArrayDumpTests() - addSliceDumpTests() - addStringDumpTests() - addInterfaceDumpTests() - addMapDumpTests() - addStructDumpTests() - addUintptrDumpTests() - addUnsafePointerDumpTests() - addChanDumpTests() - addFuncDumpTests() - addCircularDumpTests() - addPanicDumpTests() - addErrorDumpTests() - addCgoDumpTests() - - t.Logf("Running %d tests", len(dumpTests)) - for i, test := range dumpTests { - buf := new(bytes.Buffer) - spew.Fdump(buf, test.in) - s := buf.String() - if testFailed(s, test.wants) { - t.Errorf("Dump #%d\n got: %s %s", i, s, stringizeWants(test.wants)) - continue - } - } -} - -func TestDumpSortedKeys(t *testing.T) { - cfg := spew.ConfigState{SortKeys: true} - s := cfg.Sdump(map[int]string{1: "1", 3: "3", 2: "2"}) - expected := "(map[int]string) (len=3) {\n(int) 1: (string) (len=1) " + - "\"1\",\n(int) 2: (string) (len=1) \"2\",\n(int) 3: (string) " + - "(len=1) \"3\"\n" + - "}\n" - if s != expected { - t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) - } - - s = cfg.Sdump(map[stringer]int{"1": 1, "3": 3, "2": 2}) - expected = "(map[spew_test.stringer]int) (len=3) {\n" + - "(spew_test.stringer) (len=1) stringer 1: (int) 1,\n" + - "(spew_test.stringer) (len=1) stringer 2: (int) 2,\n" + - "(spew_test.stringer) (len=1) stringer 3: (int) 3\n" + - "}\n" - if s != expected { - t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) - } - - s = cfg.Sdump(map[pstringer]int{pstringer("1"): 1, pstringer("3"): 3, pstringer("2"): 2}) - expected = "(map[spew_test.pstringer]int) (len=3) {\n" + - "(spew_test.pstringer) (len=1) stringer 1: (int) 1,\n" + - "(spew_test.pstringer) (len=1) stringer 2: (int) 2,\n" + - "(spew_test.pstringer) (len=1) stringer 3: (int) 3\n" + - "}\n" - if spew.UnsafeDisabled { - expected = "(map[spew_test.pstringer]int) (len=3) {\n" + - "(spew_test.pstringer) (len=1) \"1\": (int) 1,\n" + - "(spew_test.pstringer) (len=1) \"2\": (int) 2,\n" + - "(spew_test.pstringer) (len=1) \"3\": (int) 3\n" + - "}\n" - } - if s != expected { - t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) - } - - s = cfg.Sdump(map[customError]int{customError(1): 1, customError(3): 3, customError(2): 2}) - expected = "(map[spew_test.customError]int) (len=3) {\n" + - "(spew_test.customError) error: 1: (int) 1,\n" + - "(spew_test.customError) error: 2: (int) 2,\n" + - "(spew_test.customError) error: 3: (int) 3\n" + - "}\n" - if s != expected { - t.Errorf("Sorted keys mismatch:\n %v %v", s, expected) - } - -} diff --git a/vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go b/vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go deleted file mode 100644 index 18a38358..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2013 Dave Collins -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when both cgo is supported and "-tags testcgo" is added to the go test -// command line. This means the cgo tests are only added (and hence run) when -// specifially requested. This configuration is used because spew itself -// does not require cgo to run even though it does handle certain cgo types -// specially. Rather than forcing all clients to require cgo and an external -// C compiler just to run the tests, this scheme makes them optional. -// +build cgo,testcgo - -package spew_test - -import ( - "fmt" - - "github.com/davecgh/go-spew/spew/testdata" -) - -func addCgoDumpTests() { - // C char pointer. - v := testdata.GetCgoCharPointer() - nv := testdata.GetCgoNullCharPointer() - pv := &v - vcAddr := fmt.Sprintf("%p", v) - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "*testdata._Ctype_char" - vs := "116" - addDumpTest(v, "("+vt+")("+vcAddr+")("+vs+")\n") - addDumpTest(pv, "(*"+vt+")("+vAddr+"->"+vcAddr+")("+vs+")\n") - addDumpTest(&pv, "(**"+vt+")("+pvAddr+"->"+vAddr+"->"+vcAddr+")("+vs+")\n") - addDumpTest(nv, "("+vt+")()\n") - - // C char array. - v2, v2l, v2c := testdata.GetCgoCharArray() - v2Len := fmt.Sprintf("%d", v2l) - v2Cap := fmt.Sprintf("%d", v2c) - v2t := "[6]testdata._Ctype_char" - v2s := "(len=" + v2Len + " cap=" + v2Cap + ") " + - "{\n 00000000 74 65 73 74 32 00 " + - " |test2.|\n}" - addDumpTest(v2, "("+v2t+") "+v2s+"\n") - - // C unsigned char array. - v3, v3l, v3c := testdata.GetCgoUnsignedCharArray() - v3Len := fmt.Sprintf("%d", v3l) - v3Cap := fmt.Sprintf("%d", v3c) - v3t := "[6]testdata._Ctype_unsignedchar" - v3s := "(len=" + v3Len + " cap=" + v3Cap + ") " + - "{\n 00000000 74 65 73 74 33 00 " + - " |test3.|\n}" - addDumpTest(v3, "("+v3t+") "+v3s+"\n") - - // C signed char array. - v4, v4l, v4c := testdata.GetCgoSignedCharArray() - v4Len := fmt.Sprintf("%d", v4l) - v4Cap := fmt.Sprintf("%d", v4c) - v4t := "[6]testdata._Ctype_schar" - v4t2 := "testdata._Ctype_schar" - v4s := "(len=" + v4Len + " cap=" + v4Cap + ") " + - "{\n (" + v4t2 + ") 116,\n (" + v4t2 + ") 101,\n (" + v4t2 + - ") 115,\n (" + v4t2 + ") 116,\n (" + v4t2 + ") 52,\n (" + v4t2 + - ") 0\n}" - addDumpTest(v4, "("+v4t+") "+v4s+"\n") - - // C uint8_t array. - v5, v5l, v5c := testdata.GetCgoUint8tArray() - v5Len := fmt.Sprintf("%d", v5l) - v5Cap := fmt.Sprintf("%d", v5c) - v5t := "[6]testdata._Ctype_uint8_t" - v5s := "(len=" + v5Len + " cap=" + v5Cap + ") " + - "{\n 00000000 74 65 73 74 35 00 " + - " |test5.|\n}" - addDumpTest(v5, "("+v5t+") "+v5s+"\n") - - // C typedefed unsigned char array. - v6, v6l, v6c := testdata.GetCgoTypdefedUnsignedCharArray() - v6Len := fmt.Sprintf("%d", v6l) - v6Cap := fmt.Sprintf("%d", v6c) - v6t := "[6]testdata._Ctype_custom_uchar_t" - v6s := "(len=" + v6Len + " cap=" + v6Cap + ") " + - "{\n 00000000 74 65 73 74 36 00 " + - " |test6.|\n}" - addDumpTest(v6, "("+v6t+") "+v6s+"\n") -} diff --git a/vendor/github.com/davecgh/go-spew/spew/dumpnocgo_test.go b/vendor/github.com/davecgh/go-spew/spew/dumpnocgo_test.go deleted file mode 100644 index 52a0971f..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/dumpnocgo_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2013 Dave Collins -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when either cgo is not supported or "-tags testcgo" is not added to the go -// test command line. This file intentionally does not setup any cgo tests in -// this scenario. -// +build !cgo !testcgo - -package spew_test - -func addCgoDumpTests() { - // Don't add any tests for cgo since this file is only compiled when - // there should not be any cgo tests. -} diff --git a/vendor/github.com/davecgh/go-spew/spew/example_test.go b/vendor/github.com/davecgh/go-spew/spew/example_test.go deleted file mode 100644 index de6c4e30..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/example_test.go +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright (c) 2013 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew_test - -import ( - "fmt" - - "github.com/davecgh/go-spew/spew" -) - -type Flag int - -const ( - flagOne Flag = iota - flagTwo -) - -var flagStrings = map[Flag]string{ - flagOne: "flagOne", - flagTwo: "flagTwo", -} - -func (f Flag) String() string { - if s, ok := flagStrings[f]; ok { - return s - } - return fmt.Sprintf("Unknown flag (%d)", int(f)) -} - -type Bar struct { - data uintptr -} - -type Foo struct { - unexportedField Bar - ExportedField map[interface{}]interface{} -} - -// This example demonstrates how to use Dump to dump variables to stdout. -func ExampleDump() { - // The following package level declarations are assumed for this example: - /* - type Flag int - - const ( - flagOne Flag = iota - flagTwo - ) - - var flagStrings = map[Flag]string{ - flagOne: "flagOne", - flagTwo: "flagTwo", - } - - func (f Flag) String() string { - if s, ok := flagStrings[f]; ok { - return s - } - return fmt.Sprintf("Unknown flag (%d)", int(f)) - } - - type Bar struct { - data uintptr - } - - type Foo struct { - unexportedField Bar - ExportedField map[interface{}]interface{} - } - */ - - // Setup some sample data structures for the example. - bar := Bar{uintptr(0)} - s1 := Foo{bar, map[interface{}]interface{}{"one": true}} - f := Flag(5) - b := []byte{ - 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, - 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, - 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, - 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, - 0x31, 0x32, - } - - // Dump! - spew.Dump(s1, f, b) - - // Output: - // (spew_test.Foo) { - // unexportedField: (spew_test.Bar) { - // data: (uintptr) - // }, - // ExportedField: (map[interface {}]interface {}) (len=1) { - // (string) (len=3) "one": (bool) true - // } - // } - // (spew_test.Flag) Unknown flag (5) - // ([]uint8) (len=34 cap=34) { - // 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... | - // 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0| - // 00000020 31 32 |12| - // } - // -} - -// This example demonstrates how to use Printf to display a variable with a -// format string and inline formatting. -func ExamplePrintf() { - // Create a double pointer to a uint 8. - ui8 := uint8(5) - pui8 := &ui8 - ppui8 := &pui8 - - // Create a circular data type. - type circular struct { - ui8 uint8 - c *circular - } - c := circular{ui8: 1} - c.c = &c - - // Print! - spew.Printf("ppui8: %v\n", ppui8) - spew.Printf("circular: %v\n", c) - - // Output: - // ppui8: <**>5 - // circular: {1 <*>{1 <*>}} -} - -// This example demonstrates how to use a ConfigState. -func ExampleConfigState() { - // Modify the indent level of the ConfigState only. The global - // configuration is not modified. - scs := spew.ConfigState{Indent: "\t"} - - // Output using the ConfigState instance. - v := map[string]int{"one": 1} - scs.Printf("v: %v\n", v) - scs.Dump(v) - - // Output: - // v: map[one:1] - // (map[string]int) (len=1) { - // (string) (len=3) "one": (int) 1 - // } -} - -// This example demonstrates how to use ConfigState.Dump to dump variables to -// stdout -func ExampleConfigState_Dump() { - // See the top-level Dump example for details on the types used in this - // example. - - // Create two ConfigState instances with different indentation. - scs := spew.ConfigState{Indent: "\t"} - scs2 := spew.ConfigState{Indent: " "} - - // Setup some sample data structures for the example. - bar := Bar{uintptr(0)} - s1 := Foo{bar, map[interface{}]interface{}{"one": true}} - - // Dump using the ConfigState instances. - scs.Dump(s1) - scs2.Dump(s1) - - // Output: - // (spew_test.Foo) { - // unexportedField: (spew_test.Bar) { - // data: (uintptr) - // }, - // ExportedField: (map[interface {}]interface {}) (len=1) { - // (string) (len=3) "one": (bool) true - // } - // } - // (spew_test.Foo) { - // unexportedField: (spew_test.Bar) { - // data: (uintptr) - // }, - // ExportedField: (map[interface {}]interface {}) (len=1) { - // (string) (len=3) "one": (bool) true - // } - // } - // -} - -// This example demonstrates how to use ConfigState.Printf to display a variable -// with a format string and inline formatting. -func ExampleConfigState_Printf() { - // See the top-level Dump example for details on the types used in this - // example. - - // Create two ConfigState instances and modify the method handling of the - // first ConfigState only. - scs := spew.NewDefaultConfig() - scs2 := spew.NewDefaultConfig() - scs.DisableMethods = true - - // Alternatively - // scs := spew.ConfigState{Indent: " ", DisableMethods: true} - // scs2 := spew.ConfigState{Indent: " "} - - // This is of type Flag which implements a Stringer and has raw value 1. - f := flagTwo - - // Dump using the ConfigState instances. - scs.Printf("f: %v\n", f) - scs2.Printf("f: %v\n", f) - - // Output: - // f: 1 - // f: flagTwo -} diff --git a/vendor/github.com/davecgh/go-spew/spew/format.go b/vendor/github.com/davecgh/go-spew/spew/format.go index ecf3b80e..c49875ba 100644 --- a/vendor/github.com/davecgh/go-spew/spew/format.go +++ b/vendor/github.com/davecgh/go-spew/spew/format.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Dave Collins + * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/vendor/github.com/davecgh/go-spew/spew/format_test.go b/vendor/github.com/davecgh/go-spew/spew/format_test.go deleted file mode 100644 index b664b3f1..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/format_test.go +++ /dev/null @@ -1,1558 +0,0 @@ -/* - * Copyright (c) 2013 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* -Test Summary: -NOTE: For each test, a nil pointer, a single pointer and double pointer to the -base test element are also tested to ensure proper indirection across all types. - -- Max int8, int16, int32, int64, int -- Max uint8, uint16, uint32, uint64, uint -- Boolean true and false -- Standard complex64 and complex128 -- Array containing standard ints -- Array containing type with custom formatter on pointer receiver only -- Array containing interfaces -- Slice containing standard float32 values -- Slice containing type with custom formatter on pointer receiver only -- Slice containing interfaces -- Nil slice -- Standard string -- Nil interface -- Sub-interface -- Map with string keys and int vals -- Map with custom formatter type on pointer receiver only keys and vals -- Map with interface keys and values -- Map with nil interface value -- Struct with primitives -- Struct that contains another struct -- Struct that contains custom type with Stringer pointer interface via both - exported and unexported fields -- Struct that contains embedded struct and field to same struct -- Uintptr to 0 (null pointer) -- Uintptr address of real variable -- Unsafe.Pointer to 0 (null pointer) -- Unsafe.Pointer to address of real variable -- Nil channel -- Standard int channel -- Function with no params and no returns -- Function with param and no returns -- Function with multiple params and multiple returns -- Struct that is circular through self referencing -- Structs that are circular through cross referencing -- Structs that are indirectly circular -- Type that panics in its Stringer interface -- Type that has a custom Error interface -- %x passthrough with uint -- %#x passthrough with uint -- %f passthrough with precision -- %f passthrough with width and precision -- %d passthrough with width -- %q passthrough with string -*/ - -package spew_test - -import ( - "bytes" - "fmt" - "testing" - "unsafe" - - "github.com/davecgh/go-spew/spew" -) - -// formatterTest is used to describe a test to be perfomed against NewFormatter. -type formatterTest struct { - format string - in interface{} - wants []string -} - -// formatterTests houses all of the tests to be performed against NewFormatter. -var formatterTests = make([]formatterTest, 0) - -// addFormatterTest is a helper method to append the passed input and desired -// result to formatterTests. -func addFormatterTest(format string, in interface{}, wants ...string) { - test := formatterTest{format, in, wants} - formatterTests = append(formatterTests, test) -} - -func addIntFormatterTests() { - // Max int8. - v := int8(127) - nv := (*int8)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "int8" - vs := "127" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Max int16. - v2 := int16(32767) - nv2 := (*int16)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "int16" - v2s := "32767" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") - - // Max int32. - v3 := int32(2147483647) - nv3 := (*int32)(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "int32" - v3s := "2147483647" - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3s) - addFormatterTest("%v", &pv3, "<**>"+v3s) - addFormatterTest("%v", nv3, "") - addFormatterTest("%+v", v3, v3s) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%#v", v3, "("+v3t+")"+v3s) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - - // Max int64. - v4 := int64(9223372036854775807) - nv4 := (*int64)(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "int64" - v4s := "9223372036854775807" - addFormatterTest("%v", v4, v4s) - addFormatterTest("%v", pv4, "<*>"+v4s) - addFormatterTest("%v", &pv4, "<**>"+v4s) - addFormatterTest("%v", nv4, "") - addFormatterTest("%+v", v4, v4s) - addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) - addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%#v", v4, "("+v4t+")"+v4s) - addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s) - addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s) - addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") - addFormatterTest("%#+v", v4, "("+v4t+")"+v4s) - addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s) - addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s) - addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") - - // Max int. - v5 := int(2147483647) - nv5 := (*int)(nil) - pv5 := &v5 - v5Addr := fmt.Sprintf("%p", pv5) - pv5Addr := fmt.Sprintf("%p", &pv5) - v5t := "int" - v5s := "2147483647" - addFormatterTest("%v", v5, v5s) - addFormatterTest("%v", pv5, "<*>"+v5s) - addFormatterTest("%v", &pv5, "<**>"+v5s) - addFormatterTest("%v", nv5, "") - addFormatterTest("%+v", v5, v5s) - addFormatterTest("%+v", pv5, "<*>("+v5Addr+")"+v5s) - addFormatterTest("%+v", &pv5, "<**>("+pv5Addr+"->"+v5Addr+")"+v5s) - addFormatterTest("%+v", nv5, "") - addFormatterTest("%#v", v5, "("+v5t+")"+v5s) - addFormatterTest("%#v", pv5, "(*"+v5t+")"+v5s) - addFormatterTest("%#v", &pv5, "(**"+v5t+")"+v5s) - addFormatterTest("%#v", nv5, "(*"+v5t+")"+"") - addFormatterTest("%#+v", v5, "("+v5t+")"+v5s) - addFormatterTest("%#+v", pv5, "(*"+v5t+")("+v5Addr+")"+v5s) - addFormatterTest("%#+v", &pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")"+v5s) - addFormatterTest("%#+v", nv5, "(*"+v5t+")"+"") -} - -func addUintFormatterTests() { - // Max uint8. - v := uint8(255) - nv := (*uint8)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "uint8" - vs := "255" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Max uint16. - v2 := uint16(65535) - nv2 := (*uint16)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "uint16" - v2s := "65535" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") - - // Max uint32. - v3 := uint32(4294967295) - nv3 := (*uint32)(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "uint32" - v3s := "4294967295" - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3s) - addFormatterTest("%v", &pv3, "<**>"+v3s) - addFormatterTest("%v", nv3, "") - addFormatterTest("%+v", v3, v3s) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%#v", v3, "("+v3t+")"+v3s) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - - // Max uint64. - v4 := uint64(18446744073709551615) - nv4 := (*uint64)(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "uint64" - v4s := "18446744073709551615" - addFormatterTest("%v", v4, v4s) - addFormatterTest("%v", pv4, "<*>"+v4s) - addFormatterTest("%v", &pv4, "<**>"+v4s) - addFormatterTest("%v", nv4, "") - addFormatterTest("%+v", v4, v4s) - addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) - addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%#v", v4, "("+v4t+")"+v4s) - addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s) - addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s) - addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") - addFormatterTest("%#+v", v4, "("+v4t+")"+v4s) - addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s) - addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s) - addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") - - // Max uint. - v5 := uint(4294967295) - nv5 := (*uint)(nil) - pv5 := &v5 - v5Addr := fmt.Sprintf("%p", pv5) - pv5Addr := fmt.Sprintf("%p", &pv5) - v5t := "uint" - v5s := "4294967295" - addFormatterTest("%v", v5, v5s) - addFormatterTest("%v", pv5, "<*>"+v5s) - addFormatterTest("%v", &pv5, "<**>"+v5s) - addFormatterTest("%v", nv5, "") - addFormatterTest("%+v", v5, v5s) - addFormatterTest("%+v", pv5, "<*>("+v5Addr+")"+v5s) - addFormatterTest("%+v", &pv5, "<**>("+pv5Addr+"->"+v5Addr+")"+v5s) - addFormatterTest("%+v", nv5, "") - addFormatterTest("%#v", v5, "("+v5t+")"+v5s) - addFormatterTest("%#v", pv5, "(*"+v5t+")"+v5s) - addFormatterTest("%#v", &pv5, "(**"+v5t+")"+v5s) - addFormatterTest("%#v", nv5, "(*"+v5t+")"+"") - addFormatterTest("%#+v", v5, "("+v5t+")"+v5s) - addFormatterTest("%#+v", pv5, "(*"+v5t+")("+v5Addr+")"+v5s) - addFormatterTest("%#+v", &pv5, "(**"+v5t+")("+pv5Addr+"->"+v5Addr+")"+v5s) - addFormatterTest("%#v", nv5, "(*"+v5t+")"+"") -} - -func addBoolFormatterTests() { - // Boolean true. - v := bool(true) - nv := (*bool)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "bool" - vs := "true" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Boolean false. - v2 := bool(false) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "bool" - v2s := "false" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) -} - -func addFloatFormatterTests() { - // Standard float32. - v := float32(3.1415) - nv := (*float32)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "float32" - vs := "3.1415" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Standard float64. - v2 := float64(3.1415926) - nv2 := (*float64)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "float64" - v2s := "3.1415926" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") -} - -func addComplexFormatterTests() { - // Standard complex64. - v := complex(float32(6), -2) - nv := (*complex64)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "complex64" - vs := "(6-2i)" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Standard complex128. - v2 := complex(float64(-6), 2) - nv2 := (*complex128)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "complex128" - v2s := "(-6+2i)" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") -} - -func addArrayFormatterTests() { - // Array containing standard ints. - v := [3]int{1, 2, 3} - nv := (*[3]int)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "[3]int" - vs := "[1 2 3]" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Array containing type with custom formatter on pointer receiver only. - v2 := [3]pstringer{"1", "2", "3"} - nv2 := (*[3]pstringer)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "[3]spew_test.pstringer" - v2sp := "[stringer 1 stringer 2 stringer 3]" - v2s := v2sp - if spew.UnsafeDisabled { - v2s = "[1 2 3]" - } - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2sp) - addFormatterTest("%v", &pv2, "<**>"+v2sp) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2sp) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2sp) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2sp) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2sp) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2sp) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2sp) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") - - // Array containing interfaces. - v3 := [3]interface{}{"one", int(2), uint(3)} - nv3 := (*[3]interface{})(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "[3]interface {}" - v3t2 := "string" - v3t3 := "int" - v3t4 := "uint" - v3s := "[one 2 3]" - v3s2 := "[(" + v3t2 + ")one (" + v3t3 + ")2 (" + v3t4 + ")3]" - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3s) - addFormatterTest("%v", &pv3, "<**>"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%+v", v3, v3s) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%#v", v3, "("+v3t+")"+v3s2) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s2) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s2) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s2) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s2) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s2) - addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") -} - -func addSliceFormatterTests() { - // Slice containing standard float32 values. - v := []float32{3.14, 6.28, 12.56} - nv := (*[]float32)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "[]float32" - vs := "[3.14 6.28 12.56]" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Slice containing type with custom formatter on pointer receiver only. - v2 := []pstringer{"1", "2", "3"} - nv2 := (*[]pstringer)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "[]spew_test.pstringer" - v2s := "[stringer 1 stringer 2 stringer 3]" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") - - // Slice containing interfaces. - v3 := []interface{}{"one", int(2), uint(3), nil} - nv3 := (*[]interface{})(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "[]interface {}" - v3t2 := "string" - v3t3 := "int" - v3t4 := "uint" - v3t5 := "interface {}" - v3s := "[one 2 3 ]" - v3s2 := "[(" + v3t2 + ")one (" + v3t3 + ")2 (" + v3t4 + ")3 (" + v3t5 + - ")]" - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3s) - addFormatterTest("%v", &pv3, "<**>"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%+v", v3, v3s) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%#v", v3, "("+v3t+")"+v3s2) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s2) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s2) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s2) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s2) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s2) - addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") - - // Nil slice. - var v4 []int - nv4 := (*[]int)(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "[]int" - v4s := "" - addFormatterTest("%v", v4, v4s) - addFormatterTest("%v", pv4, "<*>"+v4s) - addFormatterTest("%v", &pv4, "<**>"+v4s) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%+v", v4, v4s) - addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) - addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%#v", v4, "("+v4t+")"+v4s) - addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s) - addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s) - addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") - addFormatterTest("%#+v", v4, "("+v4t+")"+v4s) - addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s) - addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s) - addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") -} - -func addStringFormatterTests() { - // Standard string. - v := "test" - nv := (*string)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "string" - vs := "test" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") -} - -func addInterfaceFormatterTests() { - // Nil interface. - var v interface{} - nv := (*interface{})(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "interface {}" - vs := "" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Sub-interface. - v2 := interface{}(uint16(65535)) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "uint16" - v2s := "65535" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) -} - -func addMapFormatterTests() { - // Map with string keys and int vals. - v := map[string]int{"one": 1, "two": 2} - nilMap := map[string]int(nil) - nv := (*map[string]int)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "map[string]int" - vs := "map[one:1 two:2]" - vs2 := "map[two:2 one:1]" - addFormatterTest("%v", v, vs, vs2) - addFormatterTest("%v", pv, "<*>"+vs, "<*>"+vs2) - addFormatterTest("%v", &pv, "<**>"+vs, "<**>"+vs2) - addFormatterTest("%+v", nilMap, "") - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs, vs2) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs, "<*>("+vAddr+")"+vs2) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs, - "<**>("+pvAddr+"->"+vAddr+")"+vs2) - addFormatterTest("%+v", nilMap, "") - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs, "("+vt+")"+vs2) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs, "(*"+vt+")"+vs2) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs, "(**"+vt+")"+vs2) - addFormatterTest("%#v", nilMap, "("+vt+")"+"") - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs, "("+vt+")"+vs2) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs, - "(*"+vt+")("+vAddr+")"+vs2) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs, - "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs2) - addFormatterTest("%#+v", nilMap, "("+vt+")"+"") - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Map with custom formatter type on pointer receiver only keys and vals. - v2 := map[pstringer]pstringer{"one": "1"} - nv2 := (*map[pstringer]pstringer)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "map[spew_test.pstringer]spew_test.pstringer" - v2s := "map[stringer one:stringer 1]" - if spew.UnsafeDisabled { - v2s = "map[one:1]" - } - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") - - // Map with interface keys and values. - v3 := map[interface{}]interface{}{"one": 1} - nv3 := (*map[interface{}]interface{})(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "map[interface {}]interface {}" - v3t1 := "string" - v3t2 := "int" - v3s := "map[one:1]" - v3s2 := "map[(" + v3t1 + ")one:(" + v3t2 + ")1]" - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3s) - addFormatterTest("%v", &pv3, "<**>"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%+v", v3, v3s) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%#v", v3, "("+v3t+")"+v3s2) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s2) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s2) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s2) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s2) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s2) - addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") - - // Map with nil interface value - v4 := map[string]interface{}{"nil": nil} - nv4 := (*map[string]interface{})(nil) - pv4 := &v4 - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "map[string]interface {}" - v4t1 := "interface {}" - v4s := "map[nil:]" - v4s2 := "map[nil:(" + v4t1 + ")]" - addFormatterTest("%v", v4, v4s) - addFormatterTest("%v", pv4, "<*>"+v4s) - addFormatterTest("%v", &pv4, "<**>"+v4s) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%+v", v4, v4s) - addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s) - addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%#v", v4, "("+v4t+")"+v4s2) - addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s2) - addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s2) - addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") - addFormatterTest("%#+v", v4, "("+v4t+")"+v4s2) - addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s2) - addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s2) - addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") -} - -func addStructFormatterTests() { - // Struct with primitives. - type s1 struct { - a int8 - b uint8 - } - v := s1{127, 255} - nv := (*s1)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.s1" - vt2 := "int8" - vt3 := "uint8" - vs := "{127 255}" - vs2 := "{a:127 b:255}" - vs3 := "{a:(" + vt2 + ")127 b:(" + vt3 + ")255}" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs2) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs2) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs2) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs3) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs3) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs3) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs3) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs3) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs3) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Struct that contains another struct. - type s2 struct { - s1 s1 - b bool - } - v2 := s2{s1{127, 255}, true} - nv2 := (*s2)(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "spew_test.s2" - v2t2 := "spew_test.s1" - v2t3 := "int8" - v2t4 := "uint8" - v2t5 := "bool" - v2s := "{{127 255} true}" - v2s2 := "{s1:{a:127 b:255} b:true}" - v2s3 := "{s1:(" + v2t2 + "){a:(" + v2t3 + ")127 b:(" + v2t4 + ")255} b:(" + - v2t5 + ")true}" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%+v", v2, v2s2) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s2) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s2) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s3) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s3) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s3) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s3) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s3) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s3) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") - - // Struct that contains custom type with Stringer pointer interface via both - // exported and unexported fields. - type s3 struct { - s pstringer - S pstringer - } - v3 := s3{"test", "test2"} - nv3 := (*s3)(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "spew_test.s3" - v3t2 := "spew_test.pstringer" - v3s := "{stringer test stringer test2}" - v3sp := v3s - v3s2 := "{s:stringer test S:stringer test2}" - v3s2p := v3s2 - v3s3 := "{s:(" + v3t2 + ")stringer test S:(" + v3t2 + ")stringer test2}" - v3s3p := v3s3 - if spew.UnsafeDisabled { - v3s = "{test test2}" - v3sp = "{test stringer test2}" - v3s2 = "{s:test S:test2}" - v3s2p = "{s:test S:stringer test2}" - v3s3 = "{s:(" + v3t2 + ")test S:(" + v3t2 + ")test2}" - v3s3p = "{s:(" + v3t2 + ")test S:(" + v3t2 + ")stringer test2}" - } - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3sp) - addFormatterTest("%v", &pv3, "<**>"+v3sp) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%+v", v3, v3s2) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s2p) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s2p) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%#v", v3, "("+v3t+")"+v3s3) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s3p) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s3p) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s3) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s3p) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s3p) - addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") - - // Struct that contains embedded struct and field to same struct. - e := embed{"embedstr"} - v4 := embedwrap{embed: &e, e: &e} - nv4 := (*embedwrap)(nil) - pv4 := &v4 - eAddr := fmt.Sprintf("%p", &e) - v4Addr := fmt.Sprintf("%p", pv4) - pv4Addr := fmt.Sprintf("%p", &pv4) - v4t := "spew_test.embedwrap" - v4t2 := "spew_test.embed" - v4t3 := "string" - v4s := "{<*>{embedstr} <*>{embedstr}}" - v4s2 := "{embed:<*>(" + eAddr + "){a:embedstr} e:<*>(" + eAddr + - "){a:embedstr}}" - v4s3 := "{embed:(*" + v4t2 + "){a:(" + v4t3 + ")embedstr} e:(*" + v4t2 + - "){a:(" + v4t3 + ")embedstr}}" - v4s4 := "{embed:(*" + v4t2 + ")(" + eAddr + "){a:(" + v4t3 + - ")embedstr} e:(*" + v4t2 + ")(" + eAddr + "){a:(" + v4t3 + ")embedstr}}" - addFormatterTest("%v", v4, v4s) - addFormatterTest("%v", pv4, "<*>"+v4s) - addFormatterTest("%v", &pv4, "<**>"+v4s) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%+v", v4, v4s2) - addFormatterTest("%+v", pv4, "<*>("+v4Addr+")"+v4s2) - addFormatterTest("%+v", &pv4, "<**>("+pv4Addr+"->"+v4Addr+")"+v4s2) - addFormatterTest("%+v", nv4, "") - addFormatterTest("%#v", v4, "("+v4t+")"+v4s3) - addFormatterTest("%#v", pv4, "(*"+v4t+")"+v4s3) - addFormatterTest("%#v", &pv4, "(**"+v4t+")"+v4s3) - addFormatterTest("%#v", nv4, "(*"+v4t+")"+"") - addFormatterTest("%#+v", v4, "("+v4t+")"+v4s4) - addFormatterTest("%#+v", pv4, "(*"+v4t+")("+v4Addr+")"+v4s4) - addFormatterTest("%#+v", &pv4, "(**"+v4t+")("+pv4Addr+"->"+v4Addr+")"+v4s4) - addFormatterTest("%#+v", nv4, "(*"+v4t+")"+"") -} - -func addUintptrFormatterTests() { - // Null pointer. - v := uintptr(0) - nv := (*uintptr)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "uintptr" - vs := "" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Address of real variable. - i := 1 - v2 := uintptr(unsafe.Pointer(&i)) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "uintptr" - v2s := fmt.Sprintf("%p", &i) - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) -} - -func addUnsafePointerFormatterTests() { - // Null pointer. - v := unsafe.Pointer(uintptr(0)) - nv := (*unsafe.Pointer)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "unsafe.Pointer" - vs := "" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Address of real variable. - i := 1 - v2 := unsafe.Pointer(&i) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "unsafe.Pointer" - v2s := fmt.Sprintf("%p", &i) - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) -} - -func addChanFormatterTests() { - // Nil channel. - var v chan int - pv := &v - nv := (*chan int)(nil) - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "chan int" - vs := "" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Real channel. - v2 := make(chan int) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "chan int" - v2s := fmt.Sprintf("%p", v2) - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) -} - -func addFuncFormatterTests() { - // Function with no params and no returns. - v := addIntFormatterTests - nv := (*func())(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "func()" - vs := fmt.Sprintf("%p", v) - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") - - // Function with param and no returns. - v2 := TestFormatter - nv2 := (*func(*testing.T))(nil) - pv2 := &v2 - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "func(*testing.T)" - v2s := fmt.Sprintf("%p", v2) - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s) - addFormatterTest("%v", &pv2, "<**>"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%+v", v2, v2s) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%+v", nv2, "") - addFormatterTest("%#v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s) - addFormatterTest("%#v", nv2, "(*"+v2t+")"+"") - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s) - addFormatterTest("%#+v", nv2, "(*"+v2t+")"+"") - - // Function with multiple params and multiple returns. - var v3 = func(i int, s string) (b bool, err error) { - return true, nil - } - nv3 := (*func(int, string) (bool, error))(nil) - pv3 := &v3 - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "func(int, string) (bool, error)" - v3s := fmt.Sprintf("%p", v3) - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3s) - addFormatterTest("%v", &pv3, "<**>"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%+v", v3, v3s) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%+v", nv3, "") - addFormatterTest("%#v", v3, "("+v3t+")"+v3s) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s) - addFormatterTest("%#v", nv3, "(*"+v3t+")"+"") - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s) - addFormatterTest("%#+v", nv3, "(*"+v3t+")"+"") -} - -func addCircularFormatterTests() { - // Struct that is circular through self referencing. - type circular struct { - c *circular - } - v := circular{nil} - v.c = &v - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.circular" - vs := "{<*>{<*>}}" - vs2 := "{<*>}" - vs3 := "{c:<*>(" + vAddr + "){c:<*>(" + vAddr + ")}}" - vs4 := "{c:<*>(" + vAddr + ")}" - vs5 := "{c:(*" + vt + "){c:(*" + vt + ")}}" - vs6 := "{c:(*" + vt + ")}" - vs7 := "{c:(*" + vt + ")(" + vAddr + "){c:(*" + vt + ")(" + vAddr + - ")}}" - vs8 := "{c:(*" + vt + ")(" + vAddr + ")}" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs2) - addFormatterTest("%v", &pv, "<**>"+vs2) - addFormatterTest("%+v", v, vs3) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs4) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs4) - addFormatterTest("%#v", v, "("+vt+")"+vs5) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs6) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs6) - addFormatterTest("%#+v", v, "("+vt+")"+vs7) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs8) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs8) - - // Structs that are circular through cross referencing. - v2 := xref1{nil} - ts2 := xref2{&v2} - v2.ps2 = &ts2 - pv2 := &v2 - ts2Addr := fmt.Sprintf("%p", &ts2) - v2Addr := fmt.Sprintf("%p", pv2) - pv2Addr := fmt.Sprintf("%p", &pv2) - v2t := "spew_test.xref1" - v2t2 := "spew_test.xref2" - v2s := "{<*>{<*>{<*>}}}" - v2s2 := "{<*>{<*>}}" - v2s3 := "{ps2:<*>(" + ts2Addr + "){ps1:<*>(" + v2Addr + "){ps2:<*>(" + - ts2Addr + ")}}}" - v2s4 := "{ps2:<*>(" + ts2Addr + "){ps1:<*>(" + v2Addr + ")}}" - v2s5 := "{ps2:(*" + v2t2 + "){ps1:(*" + v2t + "){ps2:(*" + v2t2 + - ")}}}" - v2s6 := "{ps2:(*" + v2t2 + "){ps1:(*" + v2t + ")}}" - v2s7 := "{ps2:(*" + v2t2 + ")(" + ts2Addr + "){ps1:(*" + v2t + - ")(" + v2Addr + "){ps2:(*" + v2t2 + ")(" + ts2Addr + - ")}}}" - v2s8 := "{ps2:(*" + v2t2 + ")(" + ts2Addr + "){ps1:(*" + v2t + - ")(" + v2Addr + ")}}" - addFormatterTest("%v", v2, v2s) - addFormatterTest("%v", pv2, "<*>"+v2s2) - addFormatterTest("%v", &pv2, "<**>"+v2s2) - addFormatterTest("%+v", v2, v2s3) - addFormatterTest("%+v", pv2, "<*>("+v2Addr+")"+v2s4) - addFormatterTest("%+v", &pv2, "<**>("+pv2Addr+"->"+v2Addr+")"+v2s4) - addFormatterTest("%#v", v2, "("+v2t+")"+v2s5) - addFormatterTest("%#v", pv2, "(*"+v2t+")"+v2s6) - addFormatterTest("%#v", &pv2, "(**"+v2t+")"+v2s6) - addFormatterTest("%#+v", v2, "("+v2t+")"+v2s7) - addFormatterTest("%#+v", pv2, "(*"+v2t+")("+v2Addr+")"+v2s8) - addFormatterTest("%#+v", &pv2, "(**"+v2t+")("+pv2Addr+"->"+v2Addr+")"+v2s8) - - // Structs that are indirectly circular. - v3 := indirCir1{nil} - tic2 := indirCir2{nil} - tic3 := indirCir3{&v3} - tic2.ps3 = &tic3 - v3.ps2 = &tic2 - pv3 := &v3 - tic2Addr := fmt.Sprintf("%p", &tic2) - tic3Addr := fmt.Sprintf("%p", &tic3) - v3Addr := fmt.Sprintf("%p", pv3) - pv3Addr := fmt.Sprintf("%p", &pv3) - v3t := "spew_test.indirCir1" - v3t2 := "spew_test.indirCir2" - v3t3 := "spew_test.indirCir3" - v3s := "{<*>{<*>{<*>{<*>}}}}" - v3s2 := "{<*>{<*>{<*>}}}" - v3s3 := "{ps2:<*>(" + tic2Addr + "){ps3:<*>(" + tic3Addr + "){ps1:<*>(" + - v3Addr + "){ps2:<*>(" + tic2Addr + ")}}}}" - v3s4 := "{ps2:<*>(" + tic2Addr + "){ps3:<*>(" + tic3Addr + "){ps1:<*>(" + - v3Addr + ")}}}" - v3s5 := "{ps2:(*" + v3t2 + "){ps3:(*" + v3t3 + "){ps1:(*" + v3t + - "){ps2:(*" + v3t2 + ")}}}}" - v3s6 := "{ps2:(*" + v3t2 + "){ps3:(*" + v3t3 + "){ps1:(*" + v3t + - ")}}}" - v3s7 := "{ps2:(*" + v3t2 + ")(" + tic2Addr + "){ps3:(*" + v3t3 + ")(" + - tic3Addr + "){ps1:(*" + v3t + ")(" + v3Addr + "){ps2:(*" + v3t2 + - ")(" + tic2Addr + ")}}}}" - v3s8 := "{ps2:(*" + v3t2 + ")(" + tic2Addr + "){ps3:(*" + v3t3 + ")(" + - tic3Addr + "){ps1:(*" + v3t + ")(" + v3Addr + ")}}}" - addFormatterTest("%v", v3, v3s) - addFormatterTest("%v", pv3, "<*>"+v3s2) - addFormatterTest("%v", &pv3, "<**>"+v3s2) - addFormatterTest("%+v", v3, v3s3) - addFormatterTest("%+v", pv3, "<*>("+v3Addr+")"+v3s4) - addFormatterTest("%+v", &pv3, "<**>("+pv3Addr+"->"+v3Addr+")"+v3s4) - addFormatterTest("%#v", v3, "("+v3t+")"+v3s5) - addFormatterTest("%#v", pv3, "(*"+v3t+")"+v3s6) - addFormatterTest("%#v", &pv3, "(**"+v3t+")"+v3s6) - addFormatterTest("%#+v", v3, "("+v3t+")"+v3s7) - addFormatterTest("%#+v", pv3, "(*"+v3t+")("+v3Addr+")"+v3s8) - addFormatterTest("%#+v", &pv3, "(**"+v3t+")("+pv3Addr+"->"+v3Addr+")"+v3s8) -} - -func addPanicFormatterTests() { - // Type that panics in its Stringer interface. - v := panicer(127) - nv := (*panicer)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.panicer" - vs := "(PANIC=test panic)127" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") -} - -func addErrorFormatterTests() { - // Type that has a custom Error interface. - v := customError(127) - nv := (*customError)(nil) - pv := &v - vAddr := fmt.Sprintf("%p", pv) - pvAddr := fmt.Sprintf("%p", &pv) - vt := "spew_test.customError" - vs := "error: 127" - addFormatterTest("%v", v, vs) - addFormatterTest("%v", pv, "<*>"+vs) - addFormatterTest("%v", &pv, "<**>"+vs) - addFormatterTest("%v", nv, "") - addFormatterTest("%+v", v, vs) - addFormatterTest("%+v", pv, "<*>("+vAddr+")"+vs) - addFormatterTest("%+v", &pv, "<**>("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%+v", nv, "") - addFormatterTest("%#v", v, "("+vt+")"+vs) - addFormatterTest("%#v", pv, "(*"+vt+")"+vs) - addFormatterTest("%#v", &pv, "(**"+vt+")"+vs) - addFormatterTest("%#v", nv, "(*"+vt+")"+"") - addFormatterTest("%#+v", v, "("+vt+")"+vs) - addFormatterTest("%#+v", pv, "(*"+vt+")("+vAddr+")"+vs) - addFormatterTest("%#+v", &pv, "(**"+vt+")("+pvAddr+"->"+vAddr+")"+vs) - addFormatterTest("%#+v", nv, "(*"+vt+")"+"") -} - -func addPassthroughFormatterTests() { - // %x passthrough with uint. - v := uint(4294967295) - pv := &v - vAddr := fmt.Sprintf("%x", pv) - pvAddr := fmt.Sprintf("%x", &pv) - vs := "ffffffff" - addFormatterTest("%x", v, vs) - addFormatterTest("%x", pv, vAddr) - addFormatterTest("%x", &pv, pvAddr) - - // %#x passthrough with uint. - v2 := int(2147483647) - pv2 := &v2 - v2Addr := fmt.Sprintf("%#x", pv2) - pv2Addr := fmt.Sprintf("%#x", &pv2) - v2s := "0x7fffffff" - addFormatterTest("%#x", v2, v2s) - addFormatterTest("%#x", pv2, v2Addr) - addFormatterTest("%#x", &pv2, pv2Addr) - - // %f passthrough with precision. - addFormatterTest("%.2f", 3.1415, "3.14") - addFormatterTest("%.3f", 3.1415, "3.142") - addFormatterTest("%.4f", 3.1415, "3.1415") - - // %f passthrough with width and precision. - addFormatterTest("%5.2f", 3.1415, " 3.14") - addFormatterTest("%6.3f", 3.1415, " 3.142") - addFormatterTest("%7.4f", 3.1415, " 3.1415") - - // %d passthrough with width. - addFormatterTest("%3d", 127, "127") - addFormatterTest("%4d", 127, " 127") - addFormatterTest("%5d", 127, " 127") - - // %q passthrough with string. - addFormatterTest("%q", "test", "\"test\"") -} - -// TestFormatter executes all of the tests described by formatterTests. -func TestFormatter(t *testing.T) { - // Setup tests. - addIntFormatterTests() - addUintFormatterTests() - addBoolFormatterTests() - addFloatFormatterTests() - addComplexFormatterTests() - addArrayFormatterTests() - addSliceFormatterTests() - addStringFormatterTests() - addInterfaceFormatterTests() - addMapFormatterTests() - addStructFormatterTests() - addUintptrFormatterTests() - addUnsafePointerFormatterTests() - addChanFormatterTests() - addFuncFormatterTests() - addCircularFormatterTests() - addPanicFormatterTests() - addErrorFormatterTests() - addPassthroughFormatterTests() - - t.Logf("Running %d tests", len(formatterTests)) - for i, test := range formatterTests { - buf := new(bytes.Buffer) - spew.Fprintf(buf, test.format, test.in) - s := buf.String() - if testFailed(s, test.wants) { - t.Errorf("Formatter #%d format: %s got: %s %s", i, test.format, s, - stringizeWants(test.wants)) - continue - } - } -} - -type testStruct struct { - x int -} - -func (ts testStruct) String() string { - return fmt.Sprintf("ts.%d", ts.x) -} - -type testStructP struct { - x int -} - -func (ts *testStructP) String() string { - return fmt.Sprintf("ts.%d", ts.x) -} - -func TestPrintSortedKeys(t *testing.T) { - cfg := spew.ConfigState{SortKeys: true} - s := cfg.Sprint(map[int]string{1: "1", 3: "3", 2: "2"}) - expected := "map[1:1 2:2 3:3]" - if s != expected { - t.Errorf("Sorted keys mismatch 1:\n %v %v", s, expected) - } - - s = cfg.Sprint(map[stringer]int{"1": 1, "3": 3, "2": 2}) - expected = "map[stringer 1:1 stringer 2:2 stringer 3:3]" - if s != expected { - t.Errorf("Sorted keys mismatch 2:\n %v %v", s, expected) - } - - s = cfg.Sprint(map[pstringer]int{pstringer("1"): 1, pstringer("3"): 3, pstringer("2"): 2}) - expected = "map[stringer 1:1 stringer 2:2 stringer 3:3]" - if spew.UnsafeDisabled { - expected = "map[1:1 2:2 3:3]" - } - if s != expected { - t.Errorf("Sorted keys mismatch 3:\n %v %v", s, expected) - } - - s = cfg.Sprint(map[testStruct]int{testStruct{1}: 1, testStruct{3}: 3, testStruct{2}: 2}) - expected = "map[ts.1:1 ts.2:2 ts.3:3]" - if s != expected { - t.Errorf("Sorted keys mismatch 4:\n %v %v", s, expected) - } - - if !spew.UnsafeDisabled { - s = cfg.Sprint(map[testStructP]int{testStructP{1}: 1, testStructP{3}: 3, testStructP{2}: 2}) - expected = "map[ts.1:1 ts.2:2 ts.3:3]" - if s != expected { - t.Errorf("Sorted keys mismatch 5:\n %v %v", s, expected) - } - } - - s = cfg.Sprint(map[customError]int{customError(1): 1, customError(3): 3, customError(2): 2}) - expected = "map[error: 1:1 error: 2:2 error: 3:3]" - if s != expected { - t.Errorf("Sorted keys mismatch 6:\n %v %v", s, expected) - } -} diff --git a/vendor/github.com/davecgh/go-spew/spew/internal_test.go b/vendor/github.com/davecgh/go-spew/spew/internal_test.go deleted file mode 100644 index 1069ee21..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/internal_test.go +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2013 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* -This test file is part of the spew package rather than than the spew_test -package because it needs access to internals to properly test certain cases -which are not possible via the public interface since they should never happen. -*/ - -package spew - -import ( - "bytes" - "reflect" - "testing" -) - -// dummyFmtState implements a fake fmt.State to use for testing invalid -// reflect.Value handling. This is necessary because the fmt package catches -// invalid values before invoking the formatter on them. -type dummyFmtState struct { - bytes.Buffer -} - -func (dfs *dummyFmtState) Flag(f int) bool { - if f == int('+') { - return true - } - return false -} - -func (dfs *dummyFmtState) Precision() (int, bool) { - return 0, false -} - -func (dfs *dummyFmtState) Width() (int, bool) { - return 0, false -} - -// TestInvalidReflectValue ensures the dump and formatter code handles an -// invalid reflect value properly. This needs access to internal state since it -// should never happen in real code and therefore can't be tested via the public -// API. -func TestInvalidReflectValue(t *testing.T) { - i := 1 - - // Dump invalid reflect value. - v := new(reflect.Value) - buf := new(bytes.Buffer) - d := dumpState{w: buf, cs: &Config} - d.dump(*v) - s := buf.String() - want := "" - if s != want { - t.Errorf("InvalidReflectValue #%d\n got: %s want: %s", i, s, want) - } - i++ - - // Formatter invalid reflect value. - buf2 := new(dummyFmtState) - f := formatState{value: *v, cs: &Config, fs: buf2} - f.format(*v) - s = buf2.String() - want = "" - if s != want { - t.Errorf("InvalidReflectValue #%d got: %s want: %s", i, s, want) - } -} - -// SortValues makes the internal sortValues function available to the test -// package. -func SortValues(values []reflect.Value, cs *ConfigState) { - sortValues(values, cs) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go b/vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go deleted file mode 100644 index 83e070e9..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2013-2015 Dave Collins - -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. - -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when the code is not running on Google App Engine and "-tags disableunsafe" -// is not added to the go build command line. -// +build !appengine,!disableunsafe - -/* -This test file is part of the spew package rather than than the spew_test -package because it needs access to internals to properly test certain cases -which are not possible via the public interface since they should never happen. -*/ - -package spew - -import ( - "bytes" - "reflect" - "testing" - "unsafe" -) - -// changeKind uses unsafe to intentionally change the kind of a reflect.Value to -// the maximum kind value which does not exist. This is needed to test the -// fallback code which punts to the standard fmt library for new types that -// might get added to the language. -func changeKind(v *reflect.Value, readOnly bool) { - rvf := (*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + offsetFlag)) - *rvf = *rvf | ((1< + * Copyright (c) 2013-2016 Dave Collins * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above diff --git a/vendor/github.com/davecgh/go-spew/spew/spew_test.go b/vendor/github.com/davecgh/go-spew/spew/spew_test.go deleted file mode 100644 index dbbc0856..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/spew_test.go +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright (c) 2013 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew_test - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "testing" - - "github.com/davecgh/go-spew/spew" -) - -// spewFunc is used to identify which public function of the spew package or -// ConfigState a test applies to. -type spewFunc int - -const ( - fCSFdump spewFunc = iota - fCSFprint - fCSFprintf - fCSFprintln - fCSPrint - fCSPrintln - fCSSdump - fCSSprint - fCSSprintf - fCSSprintln - fCSErrorf - fCSNewFormatter - fErrorf - fFprint - fFprintln - fPrint - fPrintln - fSdump - fSprint - fSprintf - fSprintln -) - -// Map of spewFunc values to names for pretty printing. -var spewFuncStrings = map[spewFunc]string{ - fCSFdump: "ConfigState.Fdump", - fCSFprint: "ConfigState.Fprint", - fCSFprintf: "ConfigState.Fprintf", - fCSFprintln: "ConfigState.Fprintln", - fCSSdump: "ConfigState.Sdump", - fCSPrint: "ConfigState.Print", - fCSPrintln: "ConfigState.Println", - fCSSprint: "ConfigState.Sprint", - fCSSprintf: "ConfigState.Sprintf", - fCSSprintln: "ConfigState.Sprintln", - fCSErrorf: "ConfigState.Errorf", - fCSNewFormatter: "ConfigState.NewFormatter", - fErrorf: "spew.Errorf", - fFprint: "spew.Fprint", - fFprintln: "spew.Fprintln", - fPrint: "spew.Print", - fPrintln: "spew.Println", - fSdump: "spew.Sdump", - fSprint: "spew.Sprint", - fSprintf: "spew.Sprintf", - fSprintln: "spew.Sprintln", -} - -func (f spewFunc) String() string { - if s, ok := spewFuncStrings[f]; ok { - return s - } - return fmt.Sprintf("Unknown spewFunc (%d)", int(f)) -} - -// spewTest is used to describe a test to be performed against the public -// functions of the spew package or ConfigState. -type spewTest struct { - cs *spew.ConfigState - f spewFunc - format string - in interface{} - want string -} - -// spewTests houses the tests to be performed against the public functions of -// the spew package and ConfigState. -// -// These tests are only intended to ensure the public functions are exercised -// and are intentionally not exhaustive of types. The exhaustive type -// tests are handled in the dump and format tests. -var spewTests []spewTest - -// redirStdout is a helper function to return the standard output from f as a -// byte slice. -func redirStdout(f func()) ([]byte, error) { - tempFile, err := ioutil.TempFile("", "ss-test") - if err != nil { - return nil, err - } - fileName := tempFile.Name() - defer os.Remove(fileName) // Ignore error - - origStdout := os.Stdout - os.Stdout = tempFile - f() - os.Stdout = origStdout - tempFile.Close() - - return ioutil.ReadFile(fileName) -} - -func initSpewTests() { - // Config states with various settings. - scsDefault := spew.NewDefaultConfig() - scsNoMethods := &spew.ConfigState{Indent: " ", DisableMethods: true} - scsNoPmethods := &spew.ConfigState{Indent: " ", DisablePointerMethods: true} - scsMaxDepth := &spew.ConfigState{Indent: " ", MaxDepth: 1} - scsContinue := &spew.ConfigState{Indent: " ", ContinueOnMethod: true} - - // Variables for tests on types which implement Stringer interface with and - // without a pointer receiver. - ts := stringer("test") - tps := pstringer("test") - - // depthTester is used to test max depth handling for structs, array, slices - // and maps. - type depthTester struct { - ic indirCir1 - arr [1]string - slice []string - m map[string]int - } - dt := depthTester{indirCir1{nil}, [1]string{"arr"}, []string{"slice"}, - map[string]int{"one": 1}} - - // Variable for tests on types which implement error interface. - te := customError(10) - - spewTests = []spewTest{ - {scsDefault, fCSFdump, "", int8(127), "(int8) 127\n"}, - {scsDefault, fCSFprint, "", int16(32767), "32767"}, - {scsDefault, fCSFprintf, "%v", int32(2147483647), "2147483647"}, - {scsDefault, fCSFprintln, "", int(2147483647), "2147483647\n"}, - {scsDefault, fCSPrint, "", int64(9223372036854775807), "9223372036854775807"}, - {scsDefault, fCSPrintln, "", uint8(255), "255\n"}, - {scsDefault, fCSSdump, "", uint8(64), "(uint8) 64\n"}, - {scsDefault, fCSSprint, "", complex(1, 2), "(1+2i)"}, - {scsDefault, fCSSprintf, "%v", complex(float32(3), 4), "(3+4i)"}, - {scsDefault, fCSSprintln, "", complex(float64(5), 6), "(5+6i)\n"}, - {scsDefault, fCSErrorf, "%#v", uint16(65535), "(uint16)65535"}, - {scsDefault, fCSNewFormatter, "%v", uint32(4294967295), "4294967295"}, - {scsDefault, fErrorf, "%v", uint64(18446744073709551615), "18446744073709551615"}, - {scsDefault, fFprint, "", float32(3.14), "3.14"}, - {scsDefault, fFprintln, "", float64(6.28), "6.28\n"}, - {scsDefault, fPrint, "", true, "true"}, - {scsDefault, fPrintln, "", false, "false\n"}, - {scsDefault, fSdump, "", complex(-10, -20), "(complex128) (-10-20i)\n"}, - {scsDefault, fSprint, "", complex(-1, -2), "(-1-2i)"}, - {scsDefault, fSprintf, "%v", complex(float32(-3), -4), "(-3-4i)"}, - {scsDefault, fSprintln, "", complex(float64(-5), -6), "(-5-6i)\n"}, - {scsNoMethods, fCSFprint, "", ts, "test"}, - {scsNoMethods, fCSFprint, "", &ts, "<*>test"}, - {scsNoMethods, fCSFprint, "", tps, "test"}, - {scsNoMethods, fCSFprint, "", &tps, "<*>test"}, - {scsNoPmethods, fCSFprint, "", ts, "stringer test"}, - {scsNoPmethods, fCSFprint, "", &ts, "<*>stringer test"}, - {scsNoPmethods, fCSFprint, "", tps, "test"}, - {scsNoPmethods, fCSFprint, "", &tps, "<*>stringer test"}, - {scsMaxDepth, fCSFprint, "", dt, "{{} [] [] map[]}"}, - {scsMaxDepth, fCSFdump, "", dt, "(spew_test.depthTester) {\n" + - " ic: (spew_test.indirCir1) {\n \n },\n" + - " arr: ([1]string) (len=1 cap=1) {\n \n },\n" + - " slice: ([]string) (len=1 cap=1) {\n \n },\n" + - " m: (map[string]int) (len=1) {\n \n }\n}\n"}, - {scsContinue, fCSFprint, "", ts, "(stringer test) test"}, - {scsContinue, fCSFdump, "", ts, "(spew_test.stringer) " + - "(len=4) (stringer test) \"test\"\n"}, - {scsContinue, fCSFprint, "", te, "(error: 10) 10"}, - {scsContinue, fCSFdump, "", te, "(spew_test.customError) " + - "(error: 10) 10\n"}, - } -} - -// TestSpew executes all of the tests described by spewTests. -func TestSpew(t *testing.T) { - initSpewTests() - - t.Logf("Running %d tests", len(spewTests)) - for i, test := range spewTests { - buf := new(bytes.Buffer) - switch test.f { - case fCSFdump: - test.cs.Fdump(buf, test.in) - - case fCSFprint: - test.cs.Fprint(buf, test.in) - - case fCSFprintf: - test.cs.Fprintf(buf, test.format, test.in) - - case fCSFprintln: - test.cs.Fprintln(buf, test.in) - - case fCSPrint: - b, err := redirStdout(func() { test.cs.Print(test.in) }) - if err != nil { - t.Errorf("%v #%d %v", test.f, i, err) - continue - } - buf.Write(b) - - case fCSPrintln: - b, err := redirStdout(func() { test.cs.Println(test.in) }) - if err != nil { - t.Errorf("%v #%d %v", test.f, i, err) - continue - } - buf.Write(b) - - case fCSSdump: - str := test.cs.Sdump(test.in) - buf.WriteString(str) - - case fCSSprint: - str := test.cs.Sprint(test.in) - buf.WriteString(str) - - case fCSSprintf: - str := test.cs.Sprintf(test.format, test.in) - buf.WriteString(str) - - case fCSSprintln: - str := test.cs.Sprintln(test.in) - buf.WriteString(str) - - case fCSErrorf: - err := test.cs.Errorf(test.format, test.in) - buf.WriteString(err.Error()) - - case fCSNewFormatter: - fmt.Fprintf(buf, test.format, test.cs.NewFormatter(test.in)) - - case fErrorf: - err := spew.Errorf(test.format, test.in) - buf.WriteString(err.Error()) - - case fFprint: - spew.Fprint(buf, test.in) - - case fFprintln: - spew.Fprintln(buf, test.in) - - case fPrint: - b, err := redirStdout(func() { spew.Print(test.in) }) - if err != nil { - t.Errorf("%v #%d %v", test.f, i, err) - continue - } - buf.Write(b) - - case fPrintln: - b, err := redirStdout(func() { spew.Println(test.in) }) - if err != nil { - t.Errorf("%v #%d %v", test.f, i, err) - continue - } - buf.Write(b) - - case fSdump: - str := spew.Sdump(test.in) - buf.WriteString(str) - - case fSprint: - str := spew.Sprint(test.in) - buf.WriteString(str) - - case fSprintf: - str := spew.Sprintf(test.format, test.in) - buf.WriteString(str) - - case fSprintln: - str := spew.Sprintln(test.in) - buf.WriteString(str) - - default: - t.Errorf("%v #%d unrecognized function", test.f, i) - continue - } - s := buf.String() - if test.want != s { - t.Errorf("ConfigState #%d\n got: %s want: %s", i, s, test.want) - continue - } - } -} diff --git a/vendor/github.com/davecgh/go-spew/spew/testdata/dumpcgo.go b/vendor/github.com/davecgh/go-spew/spew/testdata/dumpcgo.go deleted file mode 100644 index 5c87dd45..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/testdata/dumpcgo.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2013 Dave Collins -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when both cgo is supported and "-tags testcgo" is added to the go test -// command line. This code should really only be in the dumpcgo_test.go file, -// but unfortunately Go will not allow cgo in test files, so this is a -// workaround to allow cgo types to be tested. This configuration is used -// because spew itself does not require cgo to run even though it does handle -// certain cgo types specially. Rather than forcing all clients to require cgo -// and an external C compiler just to run the tests, this scheme makes them -// optional. -// +build cgo,testcgo - -package testdata - -/* -#include -typedef unsigned char custom_uchar_t; - -char *ncp = 0; -char *cp = "test"; -char ca[6] = {'t', 'e', 's', 't', '2', '\0'}; -unsigned char uca[6] = {'t', 'e', 's', 't', '3', '\0'}; -signed char sca[6] = {'t', 'e', 's', 't', '4', '\0'}; -uint8_t ui8ta[6] = {'t', 'e', 's', 't', '5', '\0'}; -custom_uchar_t tuca[6] = {'t', 'e', 's', 't', '6', '\0'}; -*/ -import "C" - -// GetCgoNullCharPointer returns a null char pointer via cgo. This is only -// used for tests. -func GetCgoNullCharPointer() interface{} { - return C.ncp -} - -// GetCgoCharPointer returns a char pointer via cgo. This is only used for -// tests. -func GetCgoCharPointer() interface{} { - return C.cp -} - -// GetCgoCharArray returns a char array via cgo and the array's len and cap. -// This is only used for tests. -func GetCgoCharArray() (interface{}, int, int) { - return C.ca, len(C.ca), cap(C.ca) -} - -// GetCgoUnsignedCharArray returns an unsigned char array via cgo and the -// array's len and cap. This is only used for tests. -func GetCgoUnsignedCharArray() (interface{}, int, int) { - return C.uca, len(C.uca), cap(C.uca) -} - -// GetCgoSignedCharArray returns a signed char array via cgo and the array's len -// and cap. This is only used for tests. -func GetCgoSignedCharArray() (interface{}, int, int) { - return C.sca, len(C.sca), cap(C.sca) -} - -// GetCgoUint8tArray returns a uint8_t array via cgo and the array's len and -// cap. This is only used for tests. -func GetCgoUint8tArray() (interface{}, int, int) { - return C.ui8ta, len(C.ui8ta), cap(C.ui8ta) -} - -// GetCgoTypdefedUnsignedCharArray returns a typedefed unsigned char array via -// cgo and the array's len and cap. This is only used for tests. -func GetCgoTypdefedUnsignedCharArray() (interface{}, int, int) { - return C.tuca, len(C.tuca), cap(C.tuca) -} diff --git a/vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md b/vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md new file mode 100644 index 00000000..7fc1f793 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/MIGRATION_GUIDE.md @@ -0,0 +1,97 @@ +## Migration Guide from v2 -> v3 + +Version 3 adds several new, frequently requested features. To do so, it introduces a few breaking changes. We've worked to keep these as minimal as possible. This guide explains the breaking changes and how you can quickly update your code. + +### `Token.Claims` is now an interface type + +The most requested feature from the 2.0 verison of this library was the ability to provide a custom type to the JSON parser for claims. This was implemented by introducing a new interface, `Claims`, to replace `map[string]interface{}`. We also included two concrete implementations of `Claims`: `MapClaims` and `StandardClaims`. + +`MapClaims` is an alias for `map[string]interface{}` with built in validation behavior. It is the default claims type when using `Parse`. The usage is unchanged except you must type cast the claims property. + +The old example for parsing a token looked like this.. + +```go + if token, err := jwt.Parse(tokenString, keyLookupFunc); err == nil { + fmt.Printf("Token for user %v expires %v", token.Claims["user"], token.Claims["exp"]) + } +``` + +is now directly mapped to... + +```go + if token, err := jwt.Parse(tokenString, keyLookupFunc); err == nil { + claims := token.Claims.(jwt.MapClaims) + fmt.Printf("Token for user %v expires %v", claims["user"], claims["exp"]) + } +``` + +`StandardClaims` is designed to be embedded in your custom type. You can supply a custom claims type with the new `ParseWithClaims` function. Here's an example of using a custom claims type. + +```go + type MyCustomClaims struct { + User string + *StandardClaims + } + + if token, err := jwt.ParseWithClaims(tokenString, &MyCustomClaims{}, keyLookupFunc); err == nil { + claims := token.Claims.(*MyCustomClaims) + fmt.Printf("Token for user %v expires %v", claims.User, claims.StandardClaims.ExpiresAt) + } +``` + +### `ParseFromRequest` has been moved + +To keep this library focused on the tokens without becoming overburdened with complex request processing logic, `ParseFromRequest` and its new companion `ParseFromRequestWithClaims` have been moved to a subpackage, `request`. The method signatues have also been augmented to receive a new argument: `Extractor`. + +`Extractors` do the work of picking the token string out of a request. The interface is simple and composable. + +This simple parsing example: + +```go + if token, err := jwt.ParseFromRequest(tokenString, req, keyLookupFunc); err == nil { + fmt.Printf("Token for user %v expires %v", token.Claims["user"], token.Claims["exp"]) + } +``` + +is directly mapped to: + +```go + if token, err := request.ParseFromRequest(req, request.OAuth2Extractor, keyLookupFunc); err == nil { + claims := token.Claims.(jwt.MapClaims) + fmt.Printf("Token for user %v expires %v", claims["user"], claims["exp"]) + } +``` + +There are several concrete `Extractor` types provided for your convenience: + +* `HeaderExtractor` will search a list of headers until one contains content. +* `ArgumentExtractor` will search a list of keys in request query and form arguments until one contains content. +* `MultiExtractor` will try a list of `Extractors` in order until one returns content. +* `AuthorizationHeaderExtractor` will look in the `Authorization` header for a `Bearer` token. +* `OAuth2Extractor` searches the places an OAuth2 token would be specified (per the spec): `Authorization` header and `access_token` argument +* `PostExtractionFilter` wraps an `Extractor`, allowing you to process the content before it's parsed. A simple example is stripping the `Bearer ` text from a header + + +### RSA signing methods no longer accept `[]byte` keys + +Due to a [critical vulnerability](https://auth0.com/blog/2015/03/31/critical-vulnerabilities-in-json-web-token-libraries/), we've decided the convenience of accepting `[]byte` instead of `rsa.PublicKey` or `rsa.PrivateKey` isn't worth the risk of misuse. + +To replace this behavior, we've added two helper methods: `ParseRSAPrivateKeyFromPEM(key []byte) (*rsa.PrivateKey, error)` and `ParseRSAPublicKeyFromPEM(key []byte) (*rsa.PublicKey, error)`. These are just simple helpers for unpacking PEM encoded PKCS1 and PKCS8 keys. If your keys are encoded any other way, all you need to do is convert them to the `crypto/rsa` package's types. + +```go + func keyLookupFunc(*Token) (interface{}, error) { + // Don't forget to validate the alg is what you expect: + if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { + return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) + } + + // Look up key + key, err := lookupPublicKey(token.Header["kid"]) + if err != nil { + return nil, err + } + + // Unpack key from PEM encoded PKCS8 + return jwt.ParseRSAPublicKeyFromPEM(key) + } +``` diff --git a/vendor/github.com/dgrijalva/jwt-go/README.md b/vendor/github.com/dgrijalva/jwt-go/README.md index bf0100f4..f48365fa 100644 --- a/vendor/github.com/dgrijalva/jwt-go/README.md +++ b/vendor/github.com/dgrijalva/jwt-go/README.md @@ -1,11 +1,16 @@ -A [go](http://www.golang.org) (or 'golang' for search engine friendliness) implementation of [JSON Web Tokens](http://self-issued.info/docs/draft-jones-json-web-token.html) +A [go](http://www.golang.org) (or 'golang' for search engine friendliness) implementation of [JSON Web Tokens](http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html) [![Build Status](https://travis-ci.org/dgrijalva/jwt-go.svg?branch=master)](https://travis-ci.org/dgrijalva/jwt-go) +**BREAKING CHANGES:*** Version 3.0.0 is here. It includes _a lot_ of changes including a few that break the API. We've tried to break as few things as possible, so there should just be a few type signature changes. A full list of breaking changes is available in `VERSION_HISTORY.md`. See `MIGRATION_GUIDE.md` for more information on updating your code. + **NOTICE:** A vulnerability in JWT was [recently published](https://auth0.com/blog/2015/03/31/critical-vulnerabilities-in-json-web-token-libraries/). As this library doesn't force users to validate the `alg` is what they expected, it's possible your usage is effected. There will be an update soon to remedy this, and it will likey require backwards-incompatible changes to the API. In the short term, please make sure your implementation verifies the `alg` is what you expect. + ## What the heck is a JWT? +JWT.io has [a great introduction](https://jwt.io/introduction) to JSON Web Tokens. + In short, it's a signed JSON object that does something useful (for example, authentication). It's commonly used for `Bearer` tokens in Oauth 2. A token is made of three parts, separated by `.`'s. The first two parts are JSON objects, that have been [base64url](http://tools.ietf.org/html/rfc4648) encoded. The last part is the signature, encoded the same way. The first part is called the header. It contains the necessary information for verifying the last part, the signature. For example, which encryption method was used for signing and what key was used. @@ -16,37 +21,13 @@ The part in the middle is the interesting bit. It's called the Claims and conta This library supports the parsing and verification as well as the generation and signing of JWTs. Current supported signing algorithms are HMAC SHA, RSA, RSA-PSS, and ECDSA, though hooks are present for adding your own. -## Parse and Verify +## Examples -Parsing and verifying tokens is pretty straight forward. You pass in the token and a function for looking up the key. This is done as a callback since you may need to parse the token to find out what signing method and key was used. +See [the project documentation](https://godoc.org/github.com/dgrijalva/jwt-go) for examples of usage: -```go - token, err := jwt.Parse(myToken, func(token *jwt.Token) (interface{}, error) { - // Don't forget to validate the alg is what you expect: - if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok { - return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"]) - } - return myLookupKey(token.Header["kid"]), nil - }) - - if err == nil && token.Valid { - deliverGoodness("!") - } else { - deliverUtterRejection(":(") - } -``` - -## Create a token - -```go - // Create the token - token := jwt.New(jwt.SigningMethodHS256) - // Set some claims - token.Claims["foo"] = "bar" - token.Claims["exp"] = time.Now().Add(time.Hour * 72).Unix() - // Sign and get the complete encoded token as a string - tokenString, err := token.SignedString(mySigningKey) -``` +* [Simple example of parsing and validating a token](https://godoc.org/github.com/dgrijalva/jwt-go#example-Parse--Hmac) +* [Simple example of building and signing a token](https://godoc.org/github.com/dgrijalva/jwt-go#example-New--Hmac) +* [Directory of Examples](https://godoc.org/github.com/dgrijalva/jwt-go#pkg-examples) ## Extensions @@ -54,6 +35,12 @@ This library publishes all the necessary components for adding your own signing Here's an example of an extension that integrates with the Google App Engine signing tools: https://github.com/someone1/gcp-jwt-go +## Compliance + +This library was last reviewed to comply with [RTF 7519](http://www.rfc-editor.org/info/rfc7519) dated May 2015 with a few notable differences: + +* In order to protect against accidental use of [Unsecured JWTs](http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html#UnsecuredJWT), tokens using `alg=none` will only be accepted if the constant `jwt.UnsafeAllowNoneSignatureType` is provided as the key. + ## Project Status & Versioning This library is considered production ready. Feedback and feature requests are appreciated. The API should be considered stable. There should be very few backwards-incompatible changes outside of major version updates (and only with good reason). @@ -95,4 +82,4 @@ Without going too far down the rabbit hole, here's a description of the interact Documentation can be found [on godoc.org](http://godoc.org/github.com/dgrijalva/jwt-go). -The command line utility included in this project (cmd/jwt) provides a straightforward example of token creation and parsing as well as a useful tool for debugging your own integration. For a more http centric example, see [this gist](https://gist.github.com/cryptix/45c33ecf0ae54828e63b). +The command line utility included in this project (cmd/jwt) provides a straightforward example of token creation and parsing as well as a useful tool for debugging your own integration. You'll also find several implementation examples in to documentation. diff --git a/vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md b/vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md index 9eb7ff9c..b605b450 100644 --- a/vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md +++ b/vendor/github.com/dgrijalva/jwt-go/VERSION_HISTORY.md @@ -1,5 +1,43 @@ ## `jwt-go` Version History +#### 3.0.0 + +* **Compatibility Breaking Changes**: See MIGRATION_GUIDE.md for tips on updating your code + * Dropped support for `[]byte` keys when using RSA signing methods. This convenience feature could contribute to security vulnerabilities involving mismatched key types with signing methods. + * `ParseFromRequest` has been moved to `request` subpackage and usage has changed + * The `Claims` property on `Token` is now type `Claims` instead of `map[string]interface{}`. The default value is type `MapClaims`, which is an alias to `map[string]interface{}`. This makes it possible to use a custom type when decoding claims. +* Other Additions and Changes + * Added `Claims` interface type to allow users to decode the claims into a custom type + * Added `ParseWithClaims`, which takes a third argument of type `Claims`. Use this function instead of `Parse` if you have a custom type you'd like to decode into. + * Dramatically improved the functionality and flexibility of `ParseFromRequest`, which is now in the `request` subpackage + * Added `ParseFromRequestWithClaims` which is the `FromRequest` equivalent of `ParseWithClaims` + * Added new interface type `Extractor`, which is used for extracting JWT strings from http requests. Used with `ParseFromRequest` and `ParseFromRequestWithClaims`. + * Added several new, more specific, validation errors to error type bitmask + * Moved examples from README to executable example files + * Signing method registry is now thread safe + * Added new property to `ValidationError`, which contains the raw error returned by calls made by parse/verify (such as those returned by keyfunc or json parser) + +#### 2.7.0 + +This will likely be the last backwards compatible release before 3.0.0, excluding essential bug fixes. + +* Added new option `-show` to the `jwt` command that will just output the decoded token without verifying +* Error text for expired tokens includes how long it's been expired +* Fixed incorrect error returned from `ParseRSAPublicKeyFromPEM` +* Documentation updates + +#### 2.6.0 + +* Exposed inner error within ValidationError +* Fixed validation errors when using UseJSONNumber flag +* Added several unit tests + +#### 2.5.0 + +* Added support for signing method none. You shouldn't use this. The API tries to make this clear. +* Updated/fixed some documentation +* Added more helpful error message when trying to parse tokens that begin with `BEARER ` + #### 2.4.0 * Added new type, Parser, to allow for configuration of various parsing parameters diff --git a/vendor/github.com/dgrijalva/jwt-go/claims.go b/vendor/github.com/dgrijalva/jwt-go/claims.go new file mode 100644 index 00000000..f0228f02 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/claims.go @@ -0,0 +1,134 @@ +package jwt + +import ( + "crypto/subtle" + "fmt" + "time" +) + +// For a type to be a Claims object, it must just have a Valid method that determines +// if the token is invalid for any supported reason +type Claims interface { + Valid() error +} + +// Structured version of Claims Section, as referenced at +// https://tools.ietf.org/html/rfc7519#section-4.1 +// See examples for how to use this with your own claim types +type StandardClaims struct { + Audience string `json:"aud,omitempty"` + ExpiresAt int64 `json:"exp,omitempty"` + Id string `json:"jti,omitempty"` + IssuedAt int64 `json:"iat,omitempty"` + Issuer string `json:"iss,omitempty"` + NotBefore int64 `json:"nbf,omitempty"` + Subject string `json:"sub,omitempty"` +} + +// Validates time based claims "exp, iat, nbf". +// There is no accounting for clock skew. +// As well, if any of the above claims are not in the token, it will still +// be considered a valid claim. +func (c StandardClaims) Valid() error { + vErr := new(ValidationError) + now := TimeFunc().Unix() + + // The claims below are optional, by default, so if they are set to the + // default value in Go, let's not fail the verification for them. + if c.VerifyExpiresAt(now, false) == false { + delta := time.Unix(now, 0).Sub(time.Unix(c.ExpiresAt, 0)) + vErr.Inner = fmt.Errorf("token is expired by %v", delta) + vErr.Errors |= ValidationErrorExpired + } + + if c.VerifyIssuedAt(now, false) == false { + vErr.Inner = fmt.Errorf("Token used before issued") + vErr.Errors |= ValidationErrorIssuedAt + } + + if c.VerifyNotBefore(now, false) == false { + vErr.Inner = fmt.Errorf("token is not valid yet") + vErr.Errors |= ValidationErrorNotValidYet + } + + if vErr.valid() { + return nil + } + + return vErr +} + +// Compares the aud claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyAudience(cmp string, req bool) bool { + return verifyAud(c.Audience, cmp, req) +} + +// Compares the exp claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyExpiresAt(cmp int64, req bool) bool { + return verifyExp(c.ExpiresAt, cmp, req) +} + +// Compares the iat claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyIssuedAt(cmp int64, req bool) bool { + return verifyIat(c.IssuedAt, cmp, req) +} + +// Compares the iss claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyIssuer(cmp string, req bool) bool { + return verifyIss(c.Issuer, cmp, req) +} + +// Compares the nbf claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (c *StandardClaims) VerifyNotBefore(cmp int64, req bool) bool { + return verifyNbf(c.NotBefore, cmp, req) +} + +// ----- helpers + +func verifyAud(aud string, cmp string, required bool) bool { + if aud == "" { + return !required + } + if subtle.ConstantTimeCompare([]byte(aud), []byte(cmp)) != 0 { + return true + } else { + return false + } +} + +func verifyExp(exp int64, now int64, required bool) bool { + if exp == 0 { + return !required + } + return now <= exp +} + +func verifyIat(iat int64, now int64, required bool) bool { + if iat == 0 { + return !required + } + return now >= iat +} + +func verifyIss(iss string, cmp string, required bool) bool { + if iss == "" { + return !required + } + if subtle.ConstantTimeCompare([]byte(iss), []byte(cmp)) != 0 { + return true + } else { + return false + } +} + +func verifyNbf(nbf int64, now int64, required bool) bool { + if nbf == 0 { + return !required + } + return now >= nbf +} diff --git a/vendor/github.com/dgrijalva/jwt-go/cmd/jwt/app.go b/vendor/github.com/dgrijalva/jwt-go/cmd/jwt/app.go deleted file mode 100644 index 4068a805..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/cmd/jwt/app.go +++ /dev/null @@ -1,210 +0,0 @@ -// A useful example app. You can use this to debug your tokens on the command line. -// This is also a great place to look at how you might use this library. -// -// Example usage: -// The following will create and sign a token, then verify it and output the original claims. -// echo {\"foo\":\"bar\"} | bin/jwt -key test/sample_key -alg RS256 -sign - | bin/jwt -key test/sample_key.pub -verify - -package main - -import ( - "encoding/json" - "flag" - "fmt" - "io" - "io/ioutil" - "os" - "regexp" - "strings" - - "github.com/dgrijalva/jwt-go" -) - -var ( - // Options - flagAlg = flag.String("alg", "", "signing algorithm identifier") - flagKey = flag.String("key", "", "path to key file or '-' to read from stdin") - flagCompact = flag.Bool("compact", false, "output compact JSON") - flagDebug = flag.Bool("debug", false, "print out all kinds of debug data") - - // Modes - exactly one of these is required - flagSign = flag.String("sign", "", "path to claims object to sign or '-' to read from stdin") - flagVerify = flag.String("verify", "", "path to JWT token to verify or '-' to read from stdin") -) - -func main() { - // Usage message if you ask for -help or if you mess up inputs. - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " One of the following flags is required: sign, verify\n") - flag.PrintDefaults() - } - - // Parse command line options - flag.Parse() - - // Do the thing. If something goes wrong, print error to stderr - // and exit with a non-zero status code - if err := start(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -// Figure out which thing to do and then do that -func start() error { - if *flagSign != "" { - return signToken() - } else if *flagVerify != "" { - return verifyToken() - } else { - flag.Usage() - return fmt.Errorf("None of the required flags are present. What do you want me to do?") - } -} - -// Helper func: Read input from specified file or stdin -func loadData(p string) ([]byte, error) { - if p == "" { - return nil, fmt.Errorf("No path specified") - } - - var rdr io.Reader - if p == "-" { - rdr = os.Stdin - } else { - if f, err := os.Open(p); err == nil { - rdr = f - defer f.Close() - } else { - return nil, err - } - } - return ioutil.ReadAll(rdr) -} - -// Print a json object in accordance with the prophecy (or the command line options) -func printJSON(j interface{}) error { - var out []byte - var err error - - if *flagCompact == false { - out, err = json.MarshalIndent(j, "", " ") - } else { - out, err = json.Marshal(j) - } - - if err == nil { - fmt.Println(string(out)) - } - - return err -} - -// Verify a token and output the claims. This is a great example -// of how to verify and view a token. -func verifyToken() error { - // get the token - tokData, err := loadData(*flagVerify) - if err != nil { - return fmt.Errorf("Couldn't read token: %v", err) - } - - // trim possible whitespace from token - tokData = regexp.MustCompile(`\s*$`).ReplaceAll(tokData, []byte{}) - if *flagDebug { - fmt.Fprintf(os.Stderr, "Token len: %v bytes\n", len(tokData)) - } - - // Parse the token. Load the key from command line option - token, err := jwt.Parse(string(tokData), func(t *jwt.Token) (interface{}, error) { - data, err := loadData(*flagKey) - if err != nil { - return nil, err - } - if isEs() { - return jwt.ParseECPublicKeyFromPEM(data) - } - return data, nil - }) - - // Print some debug data - if *flagDebug && token != nil { - fmt.Fprintf(os.Stderr, "Header:\n%v\n", token.Header) - fmt.Fprintf(os.Stderr, "Claims:\n%v\n", token.Claims) - } - - // Print an error if we can't parse for some reason - if err != nil { - return fmt.Errorf("Couldn't parse token: %v", err) - } - - // Is token invalid? - if !token.Valid { - return fmt.Errorf("Token is invalid") - } - - // Print the token details - if err := printJSON(token.Claims); err != nil { - return fmt.Errorf("Failed to output claims: %v", err) - } - - return nil -} - -// Create, sign, and output a token. This is a great, simple example of -// how to use this library to create and sign a token. -func signToken() error { - // get the token data from command line arguments - tokData, err := loadData(*flagSign) - if err != nil { - return fmt.Errorf("Couldn't read token: %v", err) - } else if *flagDebug { - fmt.Fprintf(os.Stderr, "Token: %v bytes", len(tokData)) - } - - // parse the JSON of the claims - var claims map[string]interface{} - if err := json.Unmarshal(tokData, &claims); err != nil { - return fmt.Errorf("Couldn't parse claims JSON: %v", err) - } - - // get the key - var key interface{} - key, err = loadData(*flagKey) - if err != nil { - return fmt.Errorf("Couldn't read key: %v", err) - } - - // get the signing alg - alg := jwt.GetSigningMethod(*flagAlg) - if alg == nil { - return fmt.Errorf("Couldn't find signing method: %v", *flagAlg) - } - - // create a new token - token := jwt.New(alg) - token.Claims = claims - - if isEs() { - if k, ok := key.([]byte); !ok { - return fmt.Errorf("Couldn't convert key data to key") - } else { - key, err = jwt.ParseECPrivateKeyFromPEM(k) - if err != nil { - return err - } - } - } - - if out, err := token.SignedString(key); err == nil { - fmt.Println(out) - } else { - return fmt.Errorf("Error signing token: %v", err) - } - - return nil -} - -func isEs() bool { - return strings.HasPrefix(*flagAlg, "ES") -} diff --git a/vendor/github.com/dgrijalva/jwt-go/ecdsa.go b/vendor/github.com/dgrijalva/jwt-go/ecdsa.go index 0518ed10..2f59a222 100644 --- a/vendor/github.com/dgrijalva/jwt-go/ecdsa.go +++ b/vendor/github.com/dgrijalva/jwt-go/ecdsa.go @@ -69,7 +69,7 @@ func (m *SigningMethodECDSA) Verify(signingString, signature string, key interfa case *ecdsa.PublicKey: ecdsaKey = k default: - return ErrInvalidKey + return ErrInvalidKeyType } if len(sig) != 2*m.KeySize { @@ -103,7 +103,7 @@ func (m *SigningMethodECDSA) Sign(signingString string, key interface{}) (string case *ecdsa.PrivateKey: ecdsaKey = k default: - return "", ErrInvalidKey + return "", ErrInvalidKeyType } // Create the hasher diff --git a/vendor/github.com/dgrijalva/jwt-go/ecdsa_test.go b/vendor/github.com/dgrijalva/jwt-go/ecdsa_test.go deleted file mode 100644 index 753047b1..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/ecdsa_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package jwt_test - -import ( - "crypto/ecdsa" - "io/ioutil" - "strings" - "testing" - - "github.com/dgrijalva/jwt-go" -) - -var ecdsaTestData = []struct { - name string - keys map[string]string - tokenString string - alg string - claims map[string]interface{} - valid bool -}{ - { - "Basic ES256", - map[string]string{"private": "test/ec256-private.pem", "public": "test/ec256-public.pem"}, - "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJmb28iOiJiYXIifQ.feG39E-bn8HXAKhzDZq7yEAPWYDhZlwTn3sePJnU9VrGMmwdXAIEyoOnrjreYlVM_Z4N13eK9-TmMTWyfKJtHQ", - "ES256", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "Basic ES384", - map[string]string{"private": "test/ec384-private.pem", "public": "test/ec384-public.pem"}, - "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzM4NCJ9.eyJmb28iOiJiYXIifQ.ngAfKMbJUh0WWubSIYe5GMsA-aHNKwFbJk_wq3lq23aPp8H2anb1rRILIzVR0gUf4a8WzDtrzmiikuPWyCS6CN4-PwdgTk-5nehC7JXqlaBZU05p3toM3nWCwm_LXcld", - "ES384", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "Basic ES512", - map[string]string{"private": "test/ec512-private.pem", "public": "test/ec512-public.pem"}, - "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzUxMiJ9.eyJmb28iOiJiYXIifQ.AAU0TvGQOcdg2OvrwY73NHKgfk26UDekh9Prz-L_iWuTBIBqOFCWwwLsRiHB1JOddfKAls5do1W0jR_F30JpVd-6AJeTjGKA4C1A1H6gIKwRY0o_tFDIydZCl_lMBMeG5VNFAjO86-WCSKwc3hqaGkq1MugPRq_qrF9AVbuEB4JPLyL5", - "ES512", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "basic ES256 invalid: foo => bar", - map[string]string{"private": "test/ec256-private.pem", "public": "test/ec256-public.pem"}, - "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIifQ.MEQCIHoSJnmGlPaVQDqacx_2XlXEhhqtWceVopjomc2PJLtdAiAUTeGPoNYxZw0z8mgOnnIcjoxRuNDVZvybRZF3wR1l8W", - "ES256", - map[string]interface{}{"foo": "bar"}, - false, - }, -} - -func TestECDSAVerify(t *testing.T) { - for _, data := range ecdsaTestData { - var err error - - key, _ := ioutil.ReadFile(data.keys["public"]) - - var ecdsaKey *ecdsa.PublicKey - if ecdsaKey, err = jwt.ParseECPublicKeyFromPEM(key); err != nil { - t.Errorf("Unable to parse ECDSA public key: %v", err) - } - - parts := strings.Split(data.tokenString, ".") - - method := jwt.GetSigningMethod(data.alg) - err = method.Verify(strings.Join(parts[0:2], "."), parts[2], ecdsaKey) - if data.valid && err != nil { - t.Errorf("[%v] Error while verifying key: %v", data.name, err) - } - if !data.valid && err == nil { - t.Errorf("[%v] Invalid key passed validation", data.name) - } - } -} - -func TestECDSASign(t *testing.T) { - for _, data := range ecdsaTestData { - var err error - key, _ := ioutil.ReadFile(data.keys["private"]) - - var ecdsaKey *ecdsa.PrivateKey - if ecdsaKey, err = jwt.ParseECPrivateKeyFromPEM(key); err != nil { - t.Errorf("Unable to parse ECDSA private key: %v", err) - } - - if data.valid { - parts := strings.Split(data.tokenString, ".") - method := jwt.GetSigningMethod(data.alg) - sig, err := method.Sign(strings.Join(parts[0:2], "."), ecdsaKey) - if err != nil { - t.Errorf("[%v] Error signing token: %v", data.name, err) - } - if sig == parts[2] { - t.Errorf("[%v] Identical signatures\nbefore:\n%v\nafter:\n%v", data.name, parts[2], sig) - } - } - } -} diff --git a/vendor/github.com/dgrijalva/jwt-go/errors.go b/vendor/github.com/dgrijalva/jwt-go/errors.go index e9e788ff..1c93024a 100644 --- a/vendor/github.com/dgrijalva/jwt-go/errors.go +++ b/vendor/github.com/dgrijalva/jwt-go/errors.go @@ -6,9 +6,9 @@ import ( // Error constants var ( - ErrInvalidKey = errors.New("key is invalid or of invalid type") - ErrHashUnavailable = errors.New("the requested hash function is unavailable") - ErrNoTokenInRequest = errors.New("no token present in request") + ErrInvalidKey = errors.New("key is invalid") + ErrInvalidKeyType = errors.New("key is of invalid type") + ErrHashUnavailable = errors.New("the requested hash function is unavailable") ) // The errors that might occur when parsing and validating a token @@ -16,28 +16,44 @@ const ( ValidationErrorMalformed uint32 = 1 << iota // Token is malformed ValidationErrorUnverifiable // Token could not be verified because of signing problems ValidationErrorSignatureInvalid // Signature validation failed - ValidationErrorExpired // Exp validation failed - ValidationErrorNotValidYet // NBF validation failed + + // Standard Claim validation errors + ValidationErrorAudience // AUD validation failed + ValidationErrorExpired // EXP validation failed + ValidationErrorIssuedAt // IAT validation failed + ValidationErrorIssuer // ISS validation failed + ValidationErrorNotValidYet // NBF validation failed + ValidationErrorId // JTI validation failed + ValidationErrorClaimsInvalid // Generic claims validation error ) +// Helper for constructing a ValidationError with a string error message +func NewValidationError(errorText string, errorFlags uint32) *ValidationError { + return &ValidationError{ + text: errorText, + Errors: errorFlags, + } +} + // The error from Parse if token is not valid type ValidationError struct { - err string + Inner error // stores the error returned by external dependencies, i.e.: KeyFunc Errors uint32 // bitfield. see ValidationError... constants + text string // errors that do not have a valid error just have text } // Validation error is an error type func (e ValidationError) Error() string { - if e.err == "" { + if e.Inner != nil { + return e.Inner.Error() + } else if e.text != "" { + return e.text + } else { return "token is invalid" } - return e.err } // No errors func (e *ValidationError) valid() bool { - if e.Errors > 0 { - return false - } - return true + return e.Errors == 0 } diff --git a/vendor/github.com/dgrijalva/jwt-go/example_test.go b/vendor/github.com/dgrijalva/jwt-go/example_test.go deleted file mode 100644 index edb48e4d..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/example_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package jwt_test - -import ( - "fmt" - "github.com/dgrijalva/jwt-go" - "time" -) - -func ExampleParse(myToken string, myLookupKey func(interface{}) (interface{}, error)) { - token, err := jwt.Parse(myToken, func(token *jwt.Token) (interface{}, error) { - return myLookupKey(token.Header["kid"]) - }) - - if err == nil && token.Valid { - fmt.Println("Your token is valid. I like your style.") - } else { - fmt.Println("This token is terrible! I cannot accept this.") - } -} - -func ExampleNew(mySigningKey []byte) (string, error) { - // Create the token - token := jwt.New(jwt.SigningMethodHS256) - // Set some claims - token.Claims["foo"] = "bar" - token.Claims["exp"] = time.Now().Add(time.Hour * 72).Unix() - // Sign and get the complete encoded token as a string - tokenString, err := token.SignedString(mySigningKey) - return tokenString, err -} - -func ExampleParse_errorChecking(myToken string, myLookupKey func(interface{}) (interface{}, error)) { - token, err := jwt.Parse(myToken, func(token *jwt.Token) (interface{}, error) { - return myLookupKey(token.Header["kid"]) - }) - - if token.Valid { - fmt.Println("You look nice today") - } else if ve, ok := err.(*jwt.ValidationError); ok { - if ve.Errors&jwt.ValidationErrorMalformed != 0 { - fmt.Println("That's not even a token") - } else if ve.Errors&(jwt.ValidationErrorExpired|jwt.ValidationErrorNotValidYet) != 0 { - // Token is either expired or not active yet - fmt.Println("Timing is everything") - } else { - fmt.Println("Couldn't handle this token:", err) - } - } else { - fmt.Println("Couldn't handle this token:", err) - } - -} diff --git a/vendor/github.com/dgrijalva/jwt-go/hmac.go b/vendor/github.com/dgrijalva/jwt-go/hmac.go index 192e625f..c2299192 100644 --- a/vendor/github.com/dgrijalva/jwt-go/hmac.go +++ b/vendor/github.com/dgrijalva/jwt-go/hmac.go @@ -49,7 +49,7 @@ func (m *SigningMethodHMAC) Verify(signingString, signature string, key interfac // Verify the key is the right type keyBytes, ok := key.([]byte) if !ok { - return ErrInvalidKey + return ErrInvalidKeyType } // Decode signature, for comparison diff --git a/vendor/github.com/dgrijalva/jwt-go/hmac_test.go b/vendor/github.com/dgrijalva/jwt-go/hmac_test.go deleted file mode 100644 index c7e114f4..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/hmac_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package jwt_test - -import ( - "github.com/dgrijalva/jwt-go" - "io/ioutil" - "strings" - "testing" -) - -var hmacTestData = []struct { - name string - tokenString string - alg string - claims map[string]interface{} - valid bool -}{ - { - "web sample", - "eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk", - "HS256", - map[string]interface{}{"iss": "joe", "exp": 1300819380, "http://example.com/is_root": true}, - true, - }, - { - "HS384", - "eyJhbGciOiJIUzM4NCIsInR5cCI6IkpXVCJ9.eyJleHAiOjEuMzAwODE5MzhlKzA5LCJodHRwOi8vZXhhbXBsZS5jb20vaXNfcm9vdCI6dHJ1ZSwiaXNzIjoiam9lIn0.KWZEuOD5lbBxZ34g7F-SlVLAQ_r5KApWNWlZIIMyQVz5Zs58a7XdNzj5_0EcNoOy", - "HS384", - map[string]interface{}{"iss": "joe", "exp": 1300819380, "http://example.com/is_root": true}, - true, - }, - { - "HS512", - "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJleHAiOjEuMzAwODE5MzhlKzA5LCJodHRwOi8vZXhhbXBsZS5jb20vaXNfcm9vdCI6dHJ1ZSwiaXNzIjoiam9lIn0.CN7YijRX6Aw1n2jyI2Id1w90ja-DEMYiWixhYCyHnrZ1VfJRaFQz1bEbjjA5Fn4CLYaUG432dEYmSbS4Saokmw", - "HS512", - map[string]interface{}{"iss": "joe", "exp": 1300819380, "http://example.com/is_root": true}, - true, - }, - { - "web sample: invalid", - "eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXo", - "HS256", - map[string]interface{}{"iss": "joe", "exp": 1300819380, "http://example.com/is_root": true}, - false, - }, -} - -// Sample data from http://tools.ietf.org/html/draft-jones-json-web-signature-04#appendix-A.1 -var hmacTestKey, _ = ioutil.ReadFile("test/hmacTestKey") - -func TestHMACVerify(t *testing.T) { - for _, data := range hmacTestData { - parts := strings.Split(data.tokenString, ".") - - method := jwt.GetSigningMethod(data.alg) - err := method.Verify(strings.Join(parts[0:2], "."), parts[2], hmacTestKey) - if data.valid && err != nil { - t.Errorf("[%v] Error while verifying key: %v", data.name, err) - } - if !data.valid && err == nil { - t.Errorf("[%v] Invalid key passed validation", data.name) - } - } -} - -func TestHMACSign(t *testing.T) { - for _, data := range hmacTestData { - if data.valid { - parts := strings.Split(data.tokenString, ".") - method := jwt.GetSigningMethod(data.alg) - sig, err := method.Sign(strings.Join(parts[0:2], "."), hmacTestKey) - if err != nil { - t.Errorf("[%v] Error signing token: %v", data.name, err) - } - if sig != parts[2] { - t.Errorf("[%v] Incorrect signature.\nwas:\n%v\nexpecting:\n%v", data.name, sig, parts[2]) - } - } - } -} - -func BenchmarkHS256Signing(b *testing.B) { - benchmarkSigning(b, jwt.SigningMethodHS256, hmacTestKey) -} - -func BenchmarkHS384Signing(b *testing.B) { - benchmarkSigning(b, jwt.SigningMethodHS384, hmacTestKey) -} - -func BenchmarkHS512Signing(b *testing.B) { - benchmarkSigning(b, jwt.SigningMethodHS512, hmacTestKey) -} diff --git a/vendor/github.com/dgrijalva/jwt-go/map_claims.go b/vendor/github.com/dgrijalva/jwt-go/map_claims.go new file mode 100644 index 00000000..291213c4 --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/map_claims.go @@ -0,0 +1,94 @@ +package jwt + +import ( + "encoding/json" + "errors" + // "fmt" +) + +// Claims type that uses the map[string]interface{} for JSON decoding +// This is the default claims type if you don't supply one +type MapClaims map[string]interface{} + +// Compares the aud claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyAudience(cmp string, req bool) bool { + aud, _ := m["aud"].(string) + return verifyAud(aud, cmp, req) +} + +// Compares the exp claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyExpiresAt(cmp int64, req bool) bool { + switch exp := m["exp"].(type) { + case float64: + return verifyExp(int64(exp), cmp, req) + case json.Number: + v, _ := exp.Int64() + return verifyExp(v, cmp, req) + } + return req == false +} + +// Compares the iat claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyIssuedAt(cmp int64, req bool) bool { + switch iat := m["iat"].(type) { + case float64: + return verifyIat(int64(iat), cmp, req) + case json.Number: + v, _ := iat.Int64() + return verifyIat(v, cmp, req) + } + return req == false +} + +// Compares the iss claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyIssuer(cmp string, req bool) bool { + iss, _ := m["iss"].(string) + return verifyIss(iss, cmp, req) +} + +// Compares the nbf claim against cmp. +// If required is false, this method will return true if the value matches or is unset +func (m MapClaims) VerifyNotBefore(cmp int64, req bool) bool { + switch nbf := m["nbf"].(type) { + case float64: + return verifyNbf(int64(nbf), cmp, req) + case json.Number: + v, _ := nbf.Int64() + return verifyNbf(v, cmp, req) + } + return req == false +} + +// Validates time based claims "exp, iat, nbf". +// There is no accounting for clock skew. +// As well, if any of the above claims are not in the token, it will still +// be considered a valid claim. +func (m MapClaims) Valid() error { + vErr := new(ValidationError) + now := TimeFunc().Unix() + + if m.VerifyExpiresAt(now, false) == false { + vErr.Inner = errors.New("Token is expired") + vErr.Errors |= ValidationErrorExpired + } + + if m.VerifyIssuedAt(now, false) == false { + vErr.Inner = errors.New("Token used before issued") + vErr.Errors |= ValidationErrorIssuedAt + } + + if m.VerifyNotBefore(now, false) == false { + vErr.Inner = errors.New("Token is not valid yet") + vErr.Errors |= ValidationErrorNotValidYet + } + + if vErr.valid() { + return nil + } + + return vErr +} diff --git a/vendor/github.com/dgrijalva/jwt-go/none.go b/vendor/github.com/dgrijalva/jwt-go/none.go new file mode 100644 index 00000000..f04d189d --- /dev/null +++ b/vendor/github.com/dgrijalva/jwt-go/none.go @@ -0,0 +1,52 @@ +package jwt + +// Implements the none signing method. This is required by the spec +// but you probably should never use it. +var SigningMethodNone *signingMethodNone + +const UnsafeAllowNoneSignatureType unsafeNoneMagicConstant = "none signing method allowed" + +var NoneSignatureTypeDisallowedError error + +type signingMethodNone struct{} +type unsafeNoneMagicConstant string + +func init() { + SigningMethodNone = &signingMethodNone{} + NoneSignatureTypeDisallowedError = NewValidationError("'none' signature type is not allowed", ValidationErrorSignatureInvalid) + + RegisterSigningMethod(SigningMethodNone.Alg(), func() SigningMethod { + return SigningMethodNone + }) +} + +func (m *signingMethodNone) Alg() string { + return "none" +} + +// Only allow 'none' alg type if UnsafeAllowNoneSignatureType is specified as the key +func (m *signingMethodNone) Verify(signingString, signature string, key interface{}) (err error) { + // Key must be UnsafeAllowNoneSignatureType to prevent accidentally + // accepting 'none' signing method + if _, ok := key.(unsafeNoneMagicConstant); !ok { + return NoneSignatureTypeDisallowedError + } + // If signing method is none, signature must be an empty string + if signature != "" { + return NewValidationError( + "'none' signing method with non-empty signature", + ValidationErrorSignatureInvalid, + ) + } + + // Accept 'none' signing method. + return nil +} + +// Only allow 'none' signing if UnsafeAllowNoneSignatureType is specified as the key +func (m *signingMethodNone) Sign(signingString string, key interface{}) (string, error) { + if _, ok := key.(unsafeNoneMagicConstant); ok { + return "", nil + } + return "", NoneSignatureTypeDisallowedError +} diff --git a/vendor/github.com/dgrijalva/jwt-go/parser.go b/vendor/github.com/dgrijalva/jwt-go/parser.go index a078404e..7bf1c4ea 100644 --- a/vendor/github.com/dgrijalva/jwt-go/parser.go +++ b/vendor/github.com/dgrijalva/jwt-go/parser.go @@ -8,53 +8,68 @@ import ( ) type Parser struct { - ValidMethods []string // If populated, only these methods will be considered valid - UseJSONNumber bool // Use JSON Number format in JSON decoder + ValidMethods []string // If populated, only these methods will be considered valid + UseJSONNumber bool // Use JSON Number format in JSON decoder + SkipClaimsValidation bool // Skip claims validation during token parsing } // Parse, validate, and return a token. // keyFunc will receive the parsed token and should return the key for validating. // If everything is kosher, err will be nil func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { + return p.ParseWithClaims(tokenString, MapClaims{}, keyFunc) +} + +func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc) (*Token, error) { parts := strings.Split(tokenString, ".") if len(parts) != 3 { - return nil, &ValidationError{err: "token contains an invalid number of segments", Errors: ValidationErrorMalformed} + return nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } var err error token := &Token{Raw: tokenString} + // parse Header var headerBytes []byte if headerBytes, err = DecodeSegment(parts[0]); err != nil { if strings.HasPrefix(strings.ToLower(tokenString), "bearer ") { - return token, &ValidationError{err: "tokenstring should not contain 'bearer '", Errors: ValidationErrorMalformed} + return token, NewValidationError("tokenstring should not contain 'bearer '", ValidationErrorMalformed) } - return token, &ValidationError{err: err.Error(), Errors: ValidationErrorMalformed} + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} } if err = json.Unmarshal(headerBytes, &token.Header); err != nil { - return token, &ValidationError{err: err.Error(), Errors: ValidationErrorMalformed} + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} } // parse Claims var claimBytes []byte + token.Claims = claims + if claimBytes, err = DecodeSegment(parts[1]); err != nil { - return token, &ValidationError{err: err.Error(), Errors: ValidationErrorMalformed} + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} } dec := json.NewDecoder(bytes.NewBuffer(claimBytes)) if p.UseJSONNumber { dec.UseNumber() } - if err = dec.Decode(&token.Claims); err != nil { - return token, &ValidationError{err: err.Error(), Errors: ValidationErrorMalformed} + // JSON Decode. Special case for map type to avoid weird pointer behavior + if c, ok := token.Claims.(MapClaims); ok { + err = dec.Decode(&c) + } else { + err = dec.Decode(&claims) + } + // Handle decode error + if err != nil { + return token, &ValidationError{Inner: err, Errors: ValidationErrorMalformed} } // Lookup signature method if method, ok := token.Header["alg"].(string); ok { if token.Method = GetSigningMethod(method); token.Method == nil { - return token, &ValidationError{err: "signing method (alg) is unavailable.", Errors: ValidationErrorUnverifiable} + return token, NewValidationError("signing method (alg) is unavailable.", ValidationErrorUnverifiable) } } else { - return token, &ValidationError{err: "signing method (alg) is unspecified.", Errors: ValidationErrorUnverifiable} + return token, NewValidationError("signing method (alg) is unspecified.", ValidationErrorUnverifiable) } // Verify signing method is in the required set @@ -69,7 +84,7 @@ func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { } if !signingMethodValid { // signing method is not in the listed set - return token, &ValidationError{err: fmt.Sprintf("signing method %v is invalid", alg), Errors: ValidationErrorSignatureInvalid} + return token, NewValidationError(fmt.Sprintf("signing method %v is invalid", alg), ValidationErrorSignatureInvalid) } } @@ -77,33 +92,33 @@ func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { var key interface{} if keyFunc == nil { // keyFunc was not provided. short circuiting validation - return token, &ValidationError{err: "no Keyfunc was provided.", Errors: ValidationErrorUnverifiable} + return token, NewValidationError("no Keyfunc was provided.", ValidationErrorUnverifiable) } if key, err = keyFunc(token); err != nil { // keyFunc returned an error - return token, &ValidationError{err: err.Error(), Errors: ValidationErrorUnverifiable} + return token, &ValidationError{Inner: err, Errors: ValidationErrorUnverifiable} } - // Check expiration times vErr := &ValidationError{} - now := TimeFunc().Unix() - if exp, ok := token.Claims["exp"].(float64); ok { - if now > int64(exp) { - vErr.err = "token is expired" - vErr.Errors |= ValidationErrorExpired - } - } - if nbf, ok := token.Claims["nbf"].(float64); ok { - if now < int64(nbf) { - vErr.err = "token is not valid yet" - vErr.Errors |= ValidationErrorNotValidYet + + // Validate Claims + if !p.SkipClaimsValidation { + if err := token.Claims.Valid(); err != nil { + + // If the Claims Valid returned an error, check if it is a validation error, + // If it was another error type, create a ValidationError with a generic ClaimsInvalid flag set + if e, ok := err.(*ValidationError); !ok { + vErr = &ValidationError{Inner: err, Errors: ValidationErrorClaimsInvalid} + } else { + vErr = e + } } } // Perform validation token.Signature = parts[2] if err = token.Method.Verify(strings.Join(parts[0:2], "."), token.Signature, key); err != nil { - vErr.err = err.Error() + vErr.Inner = err vErr.Errors |= ValidationErrorSignatureInvalid } diff --git a/vendor/github.com/dgrijalva/jwt-go/parser_test.go b/vendor/github.com/dgrijalva/jwt-go/parser_test.go deleted file mode 100644 index 9115017b..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/parser_test.go +++ /dev/null @@ -1,239 +0,0 @@ -package jwt_test - -import ( - "encoding/json" - "fmt" - "github.com/dgrijalva/jwt-go" - "io/ioutil" - "net/http" - "reflect" - "testing" - "time" -) - -var ( - jwtTestDefaultKey []byte - defaultKeyFunc jwt.Keyfunc = func(t *jwt.Token) (interface{}, error) { return jwtTestDefaultKey, nil } - emptyKeyFunc jwt.Keyfunc = func(t *jwt.Token) (interface{}, error) { return nil, nil } - errorKeyFunc jwt.Keyfunc = func(t *jwt.Token) (interface{}, error) { return nil, fmt.Errorf("error loading key") } - nilKeyFunc jwt.Keyfunc = nil -) - -var jwtTestData = []struct { - name string - tokenString string - keyfunc jwt.Keyfunc - claims map[string]interface{} - valid bool - errors uint32 - parser *jwt.Parser -}{ - { - "basic", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", - defaultKeyFunc, - map[string]interface{}{"foo": "bar"}, - true, - 0, - nil, - }, - { - "basic expired", - "", // autogen - defaultKeyFunc, - map[string]interface{}{"foo": "bar", "exp": float64(time.Now().Unix() - 100)}, - false, - jwt.ValidationErrorExpired, - nil, - }, - { - "basic nbf", - "", // autogen - defaultKeyFunc, - map[string]interface{}{"foo": "bar", "nbf": float64(time.Now().Unix() + 100)}, - false, - jwt.ValidationErrorNotValidYet, - nil, - }, - { - "expired and nbf", - "", // autogen - defaultKeyFunc, - map[string]interface{}{"foo": "bar", "nbf": float64(time.Now().Unix() + 100), "exp": float64(time.Now().Unix() - 100)}, - false, - jwt.ValidationErrorNotValidYet | jwt.ValidationErrorExpired, - nil, - }, - { - "basic invalid", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.EhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", - defaultKeyFunc, - map[string]interface{}{"foo": "bar"}, - false, - jwt.ValidationErrorSignatureInvalid, - nil, - }, - { - "basic nokeyfunc", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", - nilKeyFunc, - map[string]interface{}{"foo": "bar"}, - false, - jwt.ValidationErrorUnverifiable, - nil, - }, - { - "basic nokey", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", - emptyKeyFunc, - map[string]interface{}{"foo": "bar"}, - false, - jwt.ValidationErrorSignatureInvalid, - nil, - }, - { - "basic errorkey", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", - errorKeyFunc, - map[string]interface{}{"foo": "bar"}, - false, - jwt.ValidationErrorUnverifiable, - nil, - }, - { - "invalid signing method", - "", - defaultKeyFunc, - map[string]interface{}{"foo": "bar"}, - false, - jwt.ValidationErrorSignatureInvalid, - &jwt.Parser{ValidMethods: []string{"HS256"}}, - }, - { - "valid signing method", - "", - defaultKeyFunc, - map[string]interface{}{"foo": "bar"}, - true, - 0, - &jwt.Parser{ValidMethods: []string{"RS256", "HS256"}}, - }, - { - "JSON Number", - "", - defaultKeyFunc, - map[string]interface{}{"foo": json.Number("123.4")}, - true, - 0, - &jwt.Parser{UseJSONNumber: true}, - }, -} - -func init() { - var e error - if jwtTestDefaultKey, e = ioutil.ReadFile("test/sample_key.pub"); e != nil { - panic(e) - } -} - -func makeSample(c map[string]interface{}) string { - key, e := ioutil.ReadFile("test/sample_key") - if e != nil { - panic(e.Error()) - } - - token := jwt.New(jwt.SigningMethodRS256) - token.Claims = c - s, e := token.SignedString(key) - - if e != nil { - panic(e.Error()) - } - - return s -} - -func TestParser_Parse(t *testing.T) { - for _, data := range jwtTestData { - if data.tokenString == "" { - data.tokenString = makeSample(data.claims) - } - - var token *jwt.Token - var err error - if data.parser != nil { - token, err = data.parser.Parse(data.tokenString, data.keyfunc) - } else { - token, err = jwt.Parse(data.tokenString, data.keyfunc) - } - - if !reflect.DeepEqual(data.claims, token.Claims) { - t.Errorf("[%v] Claims mismatch. Expecting: %v Got: %v", data.name, data.claims, token.Claims) - } - if data.valid && err != nil { - t.Errorf("[%v] Error while verifying token: %T:%v", data.name, err, err) - } - if !data.valid && err == nil { - t.Errorf("[%v] Invalid token passed validation", data.name) - } - if data.errors != 0 { - if err == nil { - t.Errorf("[%v] Expecting error. Didn't get one.", data.name) - } else { - // compare the bitfield part of the error - if e := err.(*jwt.ValidationError).Errors; e != data.errors { - t.Errorf("[%v] Errors don't match expectation. %v != %v", data.name, e, data.errors) - } - } - } - if data.valid && token.Signature == "" { - t.Errorf("[%v] Signature is left unpopulated after parsing", data.name) - } - } -} - -func TestParseRequest(t *testing.T) { - // Bearer token request - for _, data := range jwtTestData { - // FIXME: custom parsers are not supported by this helper. skip tests that require them - if data.parser != nil { - t.Logf("Skipping [%v]. Custom parsers are not supported by ParseRequest", data.name) - continue - } - - if data.tokenString == "" { - data.tokenString = makeSample(data.claims) - } - - r, _ := http.NewRequest("GET", "/", nil) - r.Header.Set("Authorization", fmt.Sprintf("Bearer %v", data.tokenString)) - token, err := jwt.ParseFromRequest(r, data.keyfunc) - - if token == nil { - t.Errorf("[%v] Token was not found: %v", data.name, err) - continue - } - if !reflect.DeepEqual(data.claims, token.Claims) { - t.Errorf("[%v] Claims mismatch. Expecting: %v Got: %v", data.name, data.claims, token.Claims) - } - if data.valid && err != nil { - t.Errorf("[%v] Error while verifying token: %v", data.name, err) - } - if !data.valid && err == nil { - t.Errorf("[%v] Invalid token passed validation", data.name) - } - } -} - -// Helper method for benchmarking various methods -func benchmarkSigning(b *testing.B, method jwt.SigningMethod, key interface{}) { - t := jwt.New(method) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - if _, err := t.SignedString(key); err != nil { - b.Fatal(err) - } - } - }) - -} diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa.go b/vendor/github.com/dgrijalva/jwt-go/rsa.go index cddffced..0ae0b198 100644 --- a/vendor/github.com/dgrijalva/jwt-go/rsa.go +++ b/vendor/github.com/dgrijalva/jwt-go/rsa.go @@ -44,8 +44,7 @@ func (m *SigningMethodRSA) Alg() string { } // Implements the Verify method from SigningMethod -// For this signing method, must be either a PEM encoded PKCS1 or PKCS8 RSA public key as -// []byte, or an rsa.PublicKey structure. +// For this signing method, must be an rsa.PublicKey structure. func (m *SigningMethodRSA) Verify(signingString, signature string, key interface{}) error { var err error @@ -56,16 +55,10 @@ func (m *SigningMethodRSA) Verify(signingString, signature string, key interface } var rsaKey *rsa.PublicKey + var ok bool - switch k := key.(type) { - case []byte: - if rsaKey, err = ParseRSAPublicKeyFromPEM(k); err != nil { - return err - } - case *rsa.PublicKey: - rsaKey = k - default: - return ErrInvalidKey + if rsaKey, ok = key.(*rsa.PublicKey); !ok { + return ErrInvalidKeyType } // Create hasher @@ -80,20 +73,13 @@ func (m *SigningMethodRSA) Verify(signingString, signature string, key interface } // Implements the Sign method from SigningMethod -// For this signing method, must be either a PEM encoded PKCS1 or PKCS8 RSA private key as -// []byte, or an rsa.PrivateKey structure. +// For this signing method, must be an rsa.PrivateKey structure. func (m *SigningMethodRSA) Sign(signingString string, key interface{}) (string, error) { - var err error var rsaKey *rsa.PrivateKey + var ok bool - switch k := key.(type) { - case []byte: - if rsaKey, err = ParseRSAPrivateKeyFromPEM(k); err != nil { - return "", err - } - case *rsa.PrivateKey: - rsaKey = k - default: + // Validate type of key + if rsaKey, ok = key.(*rsa.PrivateKey); !ok { return "", ErrInvalidKey } diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa_pss.go b/vendor/github.com/dgrijalva/jwt-go/rsa_pss.go index b5b70735..10ee9db8 100644 --- a/vendor/github.com/dgrijalva/jwt-go/rsa_pss.go +++ b/vendor/github.com/dgrijalva/jwt-go/rsa_pss.go @@ -106,7 +106,7 @@ func (m *SigningMethodRSAPSS) Sign(signingString string, key interface{}) (strin case *rsa.PrivateKey: rsaKey = k default: - return "", ErrInvalidKey + return "", ErrInvalidKeyType } // Create the hasher diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa_pss_test.go b/vendor/github.com/dgrijalva/jwt-go/rsa_pss_test.go deleted file mode 100644 index 9045aaf3..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/rsa_pss_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// +build go1.4 - -package jwt_test - -import ( - "crypto/rsa" - "io/ioutil" - "strings" - "testing" - - "github.com/dgrijalva/jwt-go" -) - -var rsaPSSTestData = []struct { - name string - tokenString string - alg string - claims map[string]interface{} - valid bool -}{ - { - "Basic PS256", - "eyJhbGciOiJQUzI1NiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIifQ.PPG4xyDVY8ffp4CcxofNmsTDXsrVG2npdQuibLhJbv4ClyPTUtR5giNSvuxo03kB6I8VXVr0Y9X7UxhJVEoJOmULAwRWaUsDnIewQa101cVhMa6iR8X37kfFoiZ6NkS-c7henVkkQWu2HtotkEtQvN5hFlk8IevXXPmvZlhQhwzB1sGzGYnoi1zOfuL98d3BIjUjtlwii5w6gYG2AEEzp7HnHCsb3jIwUPdq86Oe6hIFjtBwduIK90ca4UqzARpcfwxHwVLMpatKask00AgGVI0ysdk0BLMjmLutquD03XbThHScC2C2_Pp4cHWgMzvbgLU2RYYZcZRKr46QeNgz9w", - "PS256", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "Basic PS384", - "eyJhbGciOiJQUzM4NCIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIifQ.w7-qqgj97gK4fJsq_DCqdYQiylJjzWONvD0qWWWhqEOFk2P1eDULPnqHRnjgTXoO4HAw4YIWCsZPet7nR3Xxq4ZhMqvKW8b7KlfRTb9cH8zqFvzMmybQ4jv2hKc3bXYqVow3AoR7hN_CWXI3Dv6Kd2X5xhtxRHI6IL39oTVDUQ74LACe-9t4c3QRPuj6Pq1H4FAT2E2kW_0KOc6EQhCLWEhm2Z2__OZskDC8AiPpP8Kv4k2vB7l0IKQu8Pr4RcNBlqJdq8dA5D3hk5TLxP8V5nG1Ib80MOMMqoS3FQvSLyolFX-R_jZ3-zfq6Ebsqr0yEb0AH2CfsECF7935Pa0FKQ", - "PS384", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "Basic PS512", - "eyJhbGciOiJQUzUxMiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIifQ.GX1HWGzFaJevuSLavqqFYaW8_TpvcjQ8KfC5fXiSDzSiT9UD9nB_ikSmDNyDILNdtjZLSvVKfXxZJqCfefxAtiozEDDdJthZ-F0uO4SPFHlGiXszvKeodh7BuTWRI2wL9-ZO4mFa8nq3GMeQAfo9cx11i7nfN8n2YNQ9SHGovG7_T_AvaMZB_jT6jkDHpwGR9mz7x1sycckEo6teLdHRnH_ZdlHlxqknmyTu8Odr5Xh0sJFOL8BepWbbvIIn-P161rRHHiDWFv6nhlHwZnVzjx7HQrWSGb6-s2cdLie9QL_8XaMcUpjLkfOMKkDOfHo6AvpL7Jbwi83Z2ZTHjJWB-A", - "PS512", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "basic PS256 invalid: foo => bar", - "eyJhbGciOiJQUzI1NiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIifQ.PPG4xyDVY8ffp4CcxofNmsTDXsrVG2npdQuibLhJbv4ClyPTUtR5giNSvuxo03kB6I8VXVr0Y9X7UxhJVEoJOmULAwRWaUsDnIewQa101cVhMa6iR8X37kfFoiZ6NkS-c7henVkkQWu2HtotkEtQvN5hFlk8IevXXPmvZlhQhwzB1sGzGYnoi1zOfuL98d3BIjUjtlwii5w6gYG2AEEzp7HnHCsb3jIwUPdq86Oe6hIFjtBwduIK90ca4UqzARpcfwxHwVLMpatKask00AgGVI0ysdk0BLMjmLutquD03XbThHScC2C2_Pp4cHWgMzvbgLU2RYYZcZRKr46QeNgz9W", - "PS256", - map[string]interface{}{"foo": "bar"}, - false, - }, -} - -func TestRSAPSSVerify(t *testing.T) { - var err error - - key, _ := ioutil.ReadFile("test/sample_key.pub") - var rsaPSSKey *rsa.PublicKey - if rsaPSSKey, err = jwt.ParseRSAPublicKeyFromPEM(key); err != nil { - t.Errorf("Unable to parse RSA public key: %v", err) - } - - for _, data := range rsaPSSTestData { - parts := strings.Split(data.tokenString, ".") - - method := jwt.GetSigningMethod(data.alg) - err := method.Verify(strings.Join(parts[0:2], "."), parts[2], rsaPSSKey) - if data.valid && err != nil { - t.Errorf("[%v] Error while verifying key: %v", data.name, err) - } - if !data.valid && err == nil { - t.Errorf("[%v] Invalid key passed validation", data.name) - } - } -} - -func TestRSAPSSSign(t *testing.T) { - var err error - - key, _ := ioutil.ReadFile("test/sample_key") - var rsaPSSKey *rsa.PrivateKey - if rsaPSSKey, err = jwt.ParseRSAPrivateKeyFromPEM(key); err != nil { - t.Errorf("Unable to parse RSA private key: %v", err) - } - - for _, data := range rsaPSSTestData { - if data.valid { - parts := strings.Split(data.tokenString, ".") - method := jwt.GetSigningMethod(data.alg) - sig, err := method.Sign(strings.Join(parts[0:2], "."), rsaPSSKey) - if err != nil { - t.Errorf("[%v] Error signing token: %v", data.name, err) - } - if sig == parts[2] { - t.Errorf("[%v] Signatures shouldn't match\nnew:\n%v\noriginal:\n%v", data.name, sig, parts[2]) - } - } - } -} diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa_test.go b/vendor/github.com/dgrijalva/jwt-go/rsa_test.go deleted file mode 100644 index 13ba1fcd..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/rsa_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package jwt_test - -import ( - "github.com/dgrijalva/jwt-go" - "io/ioutil" - "strings" - "testing" -) - -var rsaTestData = []struct { - name string - tokenString string - alg string - claims map[string]interface{} - valid bool -}{ - { - "Basic RS256", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", - "RS256", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "Basic RS384", - "eyJhbGciOiJSUzM4NCIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIifQ.W-jEzRfBigtCWsinvVVuldiuilzVdU5ty0MvpLaSaqK9PlAWWlDQ1VIQ_qSKzwL5IXaZkvZFJXT3yL3n7OUVu7zCNJzdwznbC8Z-b0z2lYvcklJYi2VOFRcGbJtXUqgjk2oGsiqUMUMOLP70TTefkpsgqDxbRh9CDUfpOJgW-dU7cmgaoswe3wjUAUi6B6G2YEaiuXC0XScQYSYVKIzgKXJV8Zw-7AN_DBUI4GkTpsvQ9fVVjZM9csQiEXhYekyrKu1nu_POpQonGd8yqkIyXPECNmmqH5jH4sFiF67XhD7_JpkvLziBpI-uh86evBUadmHhb9Otqw3uV3NTaXLzJw", - "RS384", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "Basic RS512", - "eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJmb28iOiJiYXIifQ.zBlLlmRrUxx4SJPUbV37Q1joRcI9EW13grnKduK3wtYKmDXbgDpF1cZ6B-2Jsm5RB8REmMiLpGms-EjXhgnyh2TSHE-9W2gA_jvshegLWtwRVDX40ODSkTb7OVuaWgiy9y7llvcknFBTIg-FnVPVpXMmeV_pvwQyhaz1SSwSPrDyxEmksz1hq7YONXhXPpGaNbMMeDTNP_1oj8DZaqTIL9TwV8_1wb2Odt_Fy58Ke2RVFijsOLdnyEAjt2n9Mxihu9i3PhNBkkxa2GbnXBfq3kzvZ_xxGGopLdHhJjcGWXO-NiwI9_tiu14NRv4L2xC0ItD9Yz68v2ZIZEp_DuzwRQ", - "RS512", - map[string]interface{}{"foo": "bar"}, - true, - }, - { - "basic invalid: foo => bar", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.EhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", - "RS256", - map[string]interface{}{"foo": "bar"}, - false, - }, -} - -func TestRSAVerify(t *testing.T) { - key, _ := ioutil.ReadFile("test/sample_key.pub") - - for _, data := range rsaTestData { - parts := strings.Split(data.tokenString, ".") - - method := jwt.GetSigningMethod(data.alg) - err := method.Verify(strings.Join(parts[0:2], "."), parts[2], key) - if data.valid && err != nil { - t.Errorf("[%v] Error while verifying key: %v", data.name, err) - } - if !data.valid && err == nil { - t.Errorf("[%v] Invalid key passed validation", data.name) - } - } -} - -func TestRSASign(t *testing.T) { - key, _ := ioutil.ReadFile("test/sample_key") - - for _, data := range rsaTestData { - if data.valid { - parts := strings.Split(data.tokenString, ".") - method := jwt.GetSigningMethod(data.alg) - sig, err := method.Sign(strings.Join(parts[0:2], "."), key) - if err != nil { - t.Errorf("[%v] Error signing token: %v", data.name, err) - } - if sig != parts[2] { - t.Errorf("[%v] Incorrect signature.\nwas:\n%v\nexpecting:\n%v", data.name, sig, parts[2]) - } - } - } -} - -func TestRSAVerifyWithPreParsedPrivateKey(t *testing.T) { - key, _ := ioutil.ReadFile("test/sample_key.pub") - parsedKey, err := jwt.ParseRSAPublicKeyFromPEM(key) - if err != nil { - t.Fatal(err) - } - testData := rsaTestData[0] - parts := strings.Split(testData.tokenString, ".") - err = jwt.SigningMethodRS256.Verify(strings.Join(parts[0:2], "."), parts[2], parsedKey) - if err != nil { - t.Errorf("[%v] Error while verifying key: %v", testData.name, err) - } -} - -func TestRSAWithPreParsedPrivateKey(t *testing.T) { - key, _ := ioutil.ReadFile("test/sample_key") - parsedKey, err := jwt.ParseRSAPrivateKeyFromPEM(key) - if err != nil { - t.Fatal(err) - } - testData := rsaTestData[0] - parts := strings.Split(testData.tokenString, ".") - sig, err := jwt.SigningMethodRS256.Sign(strings.Join(parts[0:2], "."), parsedKey) - if err != nil { - t.Errorf("[%v] Error signing token: %v", testData.name, err) - } - if sig != parts[2] { - t.Errorf("[%v] Incorrect signature.\nwas:\n%v\nexpecting:\n%v", testData.name, sig, parts[2]) - } -} - -func TestRSAKeyParsing(t *testing.T) { - key, _ := ioutil.ReadFile("test/sample_key") - pubKey, _ := ioutil.ReadFile("test/sample_key.pub") - badKey := []byte("All your base are belong to key") - - // Test parsePrivateKey - if _, e := jwt.ParseRSAPrivateKeyFromPEM(key); e != nil { - t.Errorf("Failed to parse valid private key: %v", e) - } - - if k, e := jwt.ParseRSAPrivateKeyFromPEM(pubKey); e == nil { - t.Errorf("Parsed public key as valid private key: %v", k) - } - - if k, e := jwt.ParseRSAPrivateKeyFromPEM(badKey); e == nil { - t.Errorf("Parsed invalid key as valid private key: %v", k) - } - - // Test parsePublicKey - if _, e := jwt.ParseRSAPublicKeyFromPEM(pubKey); e != nil { - t.Errorf("Failed to parse valid public key: %v", e) - } - - if k, e := jwt.ParseRSAPublicKeyFromPEM(key); e == nil { - t.Errorf("Parsed private key as valid public key: %v", k) - } - - if k, e := jwt.ParseRSAPublicKeyFromPEM(badKey); e == nil { - t.Errorf("Parsed invalid key as valid private key: %v", k) - } - -} - -func BenchmarkRS256Signing(b *testing.B) { - key, _ := ioutil.ReadFile("test/sample_key") - parsedKey, err := jwt.ParseRSAPrivateKeyFromPEM(key) - if err != nil { - b.Fatal(err) - } - - benchmarkSigning(b, jwt.SigningMethodRS256, parsedKey) -} - -func BenchmarkRS384Signing(b *testing.B) { - key, _ := ioutil.ReadFile("test/sample_key") - parsedKey, err := jwt.ParseRSAPrivateKeyFromPEM(key) - if err != nil { - b.Fatal(err) - } - - benchmarkSigning(b, jwt.SigningMethodRS384, parsedKey) -} - -func BenchmarkRS512Signing(b *testing.B) { - key, _ := ioutil.ReadFile("test/sample_key") - parsedKey, err := jwt.ParseRSAPrivateKeyFromPEM(key) - if err != nil { - b.Fatal(err) - } - - benchmarkSigning(b, jwt.SigningMethodRS512, parsedKey) -} diff --git a/vendor/github.com/dgrijalva/jwt-go/rsa_utils.go b/vendor/github.com/dgrijalva/jwt-go/rsa_utils.go index 6f3b6ff0..213a90db 100644 --- a/vendor/github.com/dgrijalva/jwt-go/rsa_utils.go +++ b/vendor/github.com/dgrijalva/jwt-go/rsa_utils.go @@ -10,6 +10,7 @@ import ( var ( ErrKeyMustBePEMEncoded = errors.New("Invalid Key: Key must be PEM encoded PKCS1 or PKCS8 private key") ErrNotRSAPrivateKey = errors.New("Key is not a valid RSA private key") + ErrNotRSAPublicKey = errors.New("Key is not a valid RSA public key") ) // Parse PEM encoded PKCS1 or PKCS8 private key @@ -61,7 +62,7 @@ func ParseRSAPublicKeyFromPEM(key []byte) (*rsa.PublicKey, error) { var pkey *rsa.PublicKey var ok bool if pkey, ok = parsedKey.(*rsa.PublicKey); !ok { - return nil, ErrNotRSAPrivateKey + return nil, ErrNotRSAPublicKey } return pkey, nil diff --git a/vendor/github.com/dgrijalva/jwt-go/signing_method.go b/vendor/github.com/dgrijalva/jwt-go/signing_method.go index 12cf0f3d..ed1f212b 100644 --- a/vendor/github.com/dgrijalva/jwt-go/signing_method.go +++ b/vendor/github.com/dgrijalva/jwt-go/signing_method.go @@ -1,6 +1,11 @@ package jwt +import ( + "sync" +) + var signingMethods = map[string]func() SigningMethod{} +var signingMethodLock = new(sync.RWMutex) // Implement SigningMethod to add new methods for signing or verifying tokens. type SigningMethod interface { @@ -12,11 +17,17 @@ type SigningMethod interface { // Register the "alg" name and a factory function for signing method. // This is typically done during init() in the method's implementation func RegisterSigningMethod(alg string, f func() SigningMethod) { + signingMethodLock.Lock() + defer signingMethodLock.Unlock() + signingMethods[alg] = f } // Get a signing method from an "alg" string func GetSigningMethod(alg string) (method SigningMethod) { + signingMethodLock.RLock() + defer signingMethodLock.RUnlock() + if methodF, ok := signingMethods[alg]; ok { method = methodF() } diff --git a/vendor/github.com/dgrijalva/jwt-go/test/ec256-private.pem b/vendor/github.com/dgrijalva/jwt-go/test/ec256-private.pem deleted file mode 100644 index a6882b3e..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/ec256-private.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIAh5qA3rmqQQuu0vbKV/+zouz/y/Iy2pLpIcWUSyImSwoAoGCCqGSM49 -AwEHoUQDQgAEYD54V/vp+54P9DXarYqx4MPcm+HKRIQzNasYSoRQHQ/6S6Ps8tpM -cT+KvIIC8W/e9k0W7Cm72M1P9jU7SLf/vg== ------END EC PRIVATE KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/test/ec256-public.pem b/vendor/github.com/dgrijalva/jwt-go/test/ec256-public.pem deleted file mode 100644 index 7191361e..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/ec256-public.pem +++ /dev/null @@ -1,4 +0,0 @@ ------BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYD54V/vp+54P9DXarYqx4MPcm+HK -RIQzNasYSoRQHQ/6S6Ps8tpMcT+KvIIC8W/e9k0W7Cm72M1P9jU7SLf/vg== ------END PUBLIC KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/test/ec384-private.pem b/vendor/github.com/dgrijalva/jwt-go/test/ec384-private.pem deleted file mode 100644 index a86c823e..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/ec384-private.pem +++ /dev/null @@ -1,6 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MIGkAgEBBDCaCvMHKhcG/qT7xsNLYnDT7sE/D+TtWIol1ROdaK1a564vx5pHbsRy -SEKcIxISi1igBwYFK4EEACKhZANiAATYa7rJaU7feLMqrAx6adZFNQOpaUH/Uylb -ZLriOLON5YFVwtVUpO1FfEXZUIQpptRPtc5ixIPY658yhBSb6irfIJUSP9aYTflJ -GKk/mDkK4t8mWBzhiD5B6jg9cEGhGgA= ------END EC PRIVATE KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/test/ec384-public.pem b/vendor/github.com/dgrijalva/jwt-go/test/ec384-public.pem deleted file mode 100644 index e80d0056..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/ec384-public.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN PUBLIC KEY----- -MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE2Gu6yWlO33izKqwMemnWRTUDqWlB/1Mp -W2S64jizjeWBVcLVVKTtRXxF2VCEKabUT7XOYsSD2OufMoQUm+oq3yCVEj/WmE35 -SRipP5g5CuLfJlgc4Yg+Qeo4PXBBoRoA ------END PUBLIC KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/test/ec512-private.pem b/vendor/github.com/dgrijalva/jwt-go/test/ec512-private.pem deleted file mode 100644 index 213afaf1..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/ec512-private.pem +++ /dev/null @@ -1,7 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MIHcAgEBBEIB0pE4uFaWRx7t03BsYlYvF1YvKaBGyvoakxnodm9ou0R9wC+sJAjH -QZZJikOg4SwNqgQ/hyrOuDK2oAVHhgVGcYmgBwYFK4EEACOhgYkDgYYABAAJXIuw -12MUzpHggia9POBFYXSxaOGKGbMjIyDI+6q7wi7LMw3HgbaOmgIqFG72o8JBQwYN -4IbXHf+f86CRY1AA2wHzbHvt6IhkCXTNxBEffa1yMUgu8n9cKKF2iLgyQKcKqW33 -8fGOw/n3Rm2Yd/EB56u2rnD29qS+nOM9eGS+gy39OQ== ------END EC PRIVATE KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/test/ec512-public.pem b/vendor/github.com/dgrijalva/jwt-go/test/ec512-public.pem deleted file mode 100644 index 02ea0220..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/ec512-public.pem +++ /dev/null @@ -1,6 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQACVyLsNdjFM6R4IImvTzgRWF0sWjh -ihmzIyMgyPuqu8IuyzMNx4G2jpoCKhRu9qPCQUMGDeCG1x3/n/OgkWNQANsB82x7 -7eiIZAl0zcQRH32tcjFILvJ/XCihdoi4MkCnCqlt9/HxjsP590ZtmHfxAeertq5w -9vakvpzjPXhkvoMt/Tk= ------END PUBLIC KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/test/hmacTestKey b/vendor/github.com/dgrijalva/jwt-go/test/hmacTestKey deleted file mode 100644 index 435b8ddb..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/hmacTestKey +++ /dev/null @@ -1 +0,0 @@ -#5K+~ew{Z(T(P.ZGwb="=.!r.O͚gЀ \ No newline at end of file diff --git a/vendor/github.com/dgrijalva/jwt-go/test/sample_key b/vendor/github.com/dgrijalva/jwt-go/test/sample_key deleted file mode 100644 index abdbade3..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/sample_key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA4f5wg5l2hKsTeNem/V41fGnJm6gOdrj8ym3rFkEU/wT8RDtn -SgFEZOQpHEgQ7JL38xUfU0Y3g6aYw9QT0hJ7mCpz9Er5qLaMXJwZxzHzAahlfA0i -cqabvJOMvQtzD6uQv6wPEyZtDTWiQi9AXwBpHssPnpYGIn20ZZuNlX2BrClciHhC -PUIIZOQn/MmqTD31jSyjoQoV7MhhMTATKJx2XrHhR+1DcKJzQBSTAGnpYVaqpsAR -ap+nwRipr3nUTuxyGohBTSmjJ2usSeQXHI3bODIRe1AuTyHceAbewn8b462yEWKA -Rdpd9AjQW5SIVPfdsz5B6GlYQ5LdYKtznTuy7wIDAQABAoIBAQCwia1k7+2oZ2d3 -n6agCAbqIE1QXfCmh41ZqJHbOY3oRQG3X1wpcGH4Gk+O+zDVTV2JszdcOt7E5dAy -MaomETAhRxB7hlIOnEN7WKm+dGNrKRvV0wDU5ReFMRHg31/Lnu8c+5BvGjZX+ky9 -POIhFFYJqwCRlopGSUIxmVj5rSgtzk3iWOQXr+ah1bjEXvlxDOWkHN6YfpV5ThdE -KdBIPGEVqa63r9n2h+qazKrtiRqJqGnOrHzOECYbRFYhexsNFz7YT02xdfSHn7gM -IvabDDP/Qp0PjE1jdouiMaFHYnLBbgvlnZW9yuVf/rpXTUq/njxIXMmvmEyyvSDn -FcFikB8pAoGBAPF77hK4m3/rdGT7X8a/gwvZ2R121aBcdPwEaUhvj/36dx596zvY -mEOjrWfZhF083/nYWE2kVquj2wjs+otCLfifEEgXcVPTnEOPO9Zg3uNSL0nNQghj -FuD3iGLTUBCtM66oTe0jLSslHe8gLGEQqyMzHOzYxNqibxcOZIe8Qt0NAoGBAO+U -I5+XWjWEgDmvyC3TrOSf/KCGjtu0TSv30ipv27bDLMrpvPmD/5lpptTFwcxvVhCs -2b+chCjlghFSWFbBULBrfci2FtliClOVMYrlNBdUSJhf3aYSG2Doe6Bgt1n2CpNn -/iu37Y3NfemZBJA7hNl4dYe+f+uzM87cdQ214+jrAoGAXA0XxX8ll2+ToOLJsaNT -OvNB9h9Uc5qK5X5w+7G7O998BN2PC/MWp8H+2fVqpXgNENpNXttkRm1hk1dych86 -EunfdPuqsX+as44oCyJGFHVBnWpm33eWQw9YqANRI+pCJzP08I5WK3osnPiwshd+ -hR54yjgfYhBFNI7B95PmEQkCgYBzFSz7h1+s34Ycr8SvxsOBWxymG5zaCsUbPsL0 -4aCgLScCHb9J+E86aVbbVFdglYa5Id7DPTL61ixhl7WZjujspeXZGSbmq0Kcnckb -mDgqkLECiOJW2NHP/j0McAkDLL4tysF8TLDO8gvuvzNC+WQ6drO2ThrypLVZQ+ry -eBIPmwKBgEZxhqa0gVvHQG/7Od69KWj4eJP28kq13RhKay8JOoN0vPmspXJo1HY3 -CKuHRG+AP579dncdUnOMvfXOtkdM4vk0+hWASBQzM9xzVcztCa+koAugjVaLS9A+ -9uQoqEeVNTckxx0S2bYevRy7hGQmUJTyQm3j1zEUR5jpdbL83Fbq ------END RSA PRIVATE KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/test/sample_key.pub b/vendor/github.com/dgrijalva/jwt-go/test/sample_key.pub deleted file mode 100644 index 03dc982a..00000000 --- a/vendor/github.com/dgrijalva/jwt-go/test/sample_key.pub +++ /dev/null @@ -1,9 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4f5wg5l2hKsTeNem/V41 -fGnJm6gOdrj8ym3rFkEU/wT8RDtnSgFEZOQpHEgQ7JL38xUfU0Y3g6aYw9QT0hJ7 -mCpz9Er5qLaMXJwZxzHzAahlfA0icqabvJOMvQtzD6uQv6wPEyZtDTWiQi9AXwBp -HssPnpYGIn20ZZuNlX2BrClciHhCPUIIZOQn/MmqTD31jSyjoQoV7MhhMTATKJx2 -XrHhR+1DcKJzQBSTAGnpYVaqpsARap+nwRipr3nUTuxyGohBTSmjJ2usSeQXHI3b -ODIRe1AuTyHceAbewn8b462yEWKARdpd9AjQW5SIVPfdsz5B6GlYQ5LdYKtznTuy -7wIDAQAB ------END PUBLIC KEY----- diff --git a/vendor/github.com/dgrijalva/jwt-go/token.go b/vendor/github.com/dgrijalva/jwt-go/token.go index 1cf267d4..d637e086 100644 --- a/vendor/github.com/dgrijalva/jwt-go/token.go +++ b/vendor/github.com/dgrijalva/jwt-go/token.go @@ -3,7 +3,6 @@ package jwt import ( "encoding/base64" "encoding/json" - "net/http" "strings" "time" ) @@ -15,7 +14,7 @@ var TimeFunc = time.Now // Parse methods use this callback function to supply // the key for verification. The function receives the parsed, -// but unverified Token. This allows you to use propries in the +// but unverified Token. This allows you to use properties in the // Header of the token (such as `kid`) to identify which key to use. type Keyfunc func(*Token) (interface{}, error) @@ -25,19 +24,23 @@ type Token struct { Raw string // The raw token. Populated when you Parse a token Method SigningMethod // The signing method used or to be used Header map[string]interface{} // The first segment of the token - Claims map[string]interface{} // The second segment of the token + Claims Claims // The second segment of the token Signature string // The third segment of the token. Populated when you Parse a token Valid bool // Is the token valid? Populated when you Parse/Verify a token } // Create a new Token. Takes a signing method func New(method SigningMethod) *Token { + return NewWithClaims(method, MapClaims{}) +} + +func NewWithClaims(method SigningMethod, claims Claims) *Token { return &Token{ Header: map[string]interface{}{ "typ": "JWT", "alg": method.Alg(), }, - Claims: make(map[string]interface{}), + Claims: claims, Method: method, } } @@ -63,16 +66,15 @@ func (t *Token) SigningString() (string, error) { var err error parts := make([]string, 2) for i, _ := range parts { - var source map[string]interface{} - if i == 0 { - source = t.Header - } else { - source = t.Claims - } - var jsonValue []byte - if jsonValue, err = json.Marshal(source); err != nil { - return "", err + if i == 0 { + if jsonValue, err = json.Marshal(t.Header); err != nil { + return "", err + } + } else { + if jsonValue, err = json.Marshal(t.Claims); err != nil { + return "", err + } } parts[i] = EncodeSegment(jsonValue) @@ -87,28 +89,8 @@ func Parse(tokenString string, keyFunc Keyfunc) (*Token, error) { return new(Parser).Parse(tokenString, keyFunc) } -// Try to find the token in an http.Request. -// This method will call ParseMultipartForm if there's no token in the header. -// Currently, it looks in the Authorization header as well as -// looking for an 'access_token' request parameter in req.Form. -func ParseFromRequest(req *http.Request, keyFunc Keyfunc) (token *Token, err error) { - - // Look for an Authorization header - if ah := req.Header.Get("Authorization"); ah != "" { - // Should be a bearer token - if len(ah) > 6 && strings.ToUpper(ah[0:7]) == "BEARER " { - return Parse(ah[7:], keyFunc) - } - } - - // Look for "access_token" parameter - req.ParseMultipartForm(10e6) - if tokStr := req.Form.Get("access_token"); tokStr != "" { - return Parse(tokStr, keyFunc) - } - - return nil, ErrNoTokenInRequest - +func ParseWithClaims(tokenString string, claims Claims, keyFunc Keyfunc) (*Token, error) { + return new(Parser).ParseWithClaims(tokenString, claims, keyFunc) } // Encode JWT specific base64url encoding with padding stripped diff --git a/vendor/github.com/dsnet/compress/LICENSE.md b/vendor/github.com/dsnet/compress/LICENSE.md new file mode 100644 index 00000000..2bd6ed1f --- /dev/null +++ b/vendor/github.com/dsnet/compress/LICENSE.md @@ -0,0 +1,24 @@ +Copyright © 2015, Joe Tsai and The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. +* Neither the copyright holder nor the names of its contributors may be used to +endorse or promote products derived from this software without specific prior +written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/dsnet/compress/brotli/brotli_test.go b/vendor/github.com/dsnet/compress/brotli/brotli_test.go deleted file mode 100644 index 511a58fe..00000000 --- a/vendor/github.com/dsnet/compress/brotli/brotli_test.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli diff --git a/vendor/github.com/dsnet/compress/brotli/common_test.go b/vendor/github.com/dsnet/compress/brotli/common_test.go deleted file mode 100644 index fb71d2f0..00000000 --- a/vendor/github.com/dsnet/compress/brotli/common_test.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli - -import "hash/crc32" -import "bytes" -import "encoding/hex" -import "testing" - -func TestTableCRC(t *testing.T) { - // Convert transformLUT to byte array according to Appendix B of the RFC. - var transformBuf bytes.Buffer - for _, t := range transformLUT { - transformBuf.WriteString(t.prefix + "\x00") - transformBuf.WriteByte(byte(t.transform)) - transformBuf.WriteString(t.suffix + "\x00") - } - - var vectors = []struct { - crc uint32 - buf []byte - }{ - {crc: 0x5136cb04, buf: dictLUT[:]}, - {crc: 0x8e91efb7, buf: contextLUT0[:]}, - {crc: 0xd01a32f4, buf: contextLUT1[:]}, - {crc: 0x0dd7a0d6, buf: contextLUT2[:]}, - {crc: 0x3d965f81, buf: transformBuf.Bytes()}, - } - - for i, v := range vectors { - crc := crc32.ChecksumIEEE(v.buf) - if crc != v.crc { - t.Errorf("test %d, CRC-32 mismatch: got %08x, want %08x", i, crc, v.crc) - } - } -} - -func TestMoveToFront(t *testing.T) { - var vectors = []struct { - input, output string - }{{ - input: "", - output: "", - }, { - input: "ff00ff00ff00ff00", - output: "ff01010101010101", - }, { - input: "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000010000000001", - output: "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000010100000001", - }, { - input: "0000000000000000000000010101010101010101010101010101010101010101" + - "0101010101010101010101010101010101010101000000000000000203030004", - output: "0000000000000000000000010000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000010000000000000203000204", - }, { - input: "00000001", - output: "00000001", - }, { - input: "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000100000000", - output: "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000101000000", - }, { - input: "0000000000000000010101010101010101010101010101010101010101010101" + - "0101010101010101010101010101010101010101010101010101010101010101" + - "0101010101010101010200000000000000020203030303030304040505050505" + - "0505050505050505050505050505050505050505050505050505050505050505", - output: "0000000000000000010000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000202000000000000010003000000000004000500000000" + - "0000000000000000000000000000000000000000000000000000000000000000", - }, { - input: "000000010101000202020003030300040404000505050006", - output: "000000010000010200000103000001040000010500000106", - }, { - input: "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000001010102020202030202020202" + - "0202020202020202020202020202020202020202020202020202020202020202", - output: "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000001000002000000030100000000" + - "0000000000000000000000000000000000000000000000000000000000000000", - }, { - input: "0000000000000000000102020202020202020000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000302020202020202010103030303030304040505050505" + - "0505050505050505050505050505050505050505050505050505050505050505", - output: "0000000000000000000102000000000000000200000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000302000000000000030002000000000004000500000000" + - "0000000000000000000000000000000000000000000000000000000000000000", - }, { - input: "0000010000000102020201030303010404020105", - output: "0000010100000102000001030000010400030205", - }, { - input: "0000000000000000010202010101010101010202020202020202020202020202" + - "0202020202020202020202020202020202020202020202020202020202020202" + - "0202020202020202020201010101010101020202020202020203040000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000201010101010101020205050505050503030606060606" + - "0606060606060606060606060606060606060606060606060606060606060606" + - "0606060606060606060202020202020202000702020202020202040404040404" + - "0404040404040404040404040404040404040404040404040404040404040404", - output: "0000000000000000010200010000000000000100000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000001000000000000010000000000000003040400000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000304000000000000010005000000000005000600000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000300000000000000050702000000000000070000000000" + - "0000000000000000000000000000000000000000000000000000000000000000", - }, { - input: "0000000000000001010100020202000303030004040400050505000606060007" + - "070700080808000909090a0a", - output: "0000000000000001000001020000010300000104000001050000010600000107" + - "000001080000010900000a00", - }, { - input: "0000010203030303040000050600070507050201070206060804000400020002" + - "06000200000006000905020000080805050a0202000808080808080105080808" + - "0400050205020505050505050b0205040b0505050505050b0605050505050505" + - "0505050505050505050505050505050502050505050505050505050202050505" + - "040502020b0b0b0b020b0b0b0b0b0b02020b0b0b0b0b0b02020b0b0b0b0b0b0b" + - "0b0b0b0b0b0b0b02020b0b0b0b0b0b0b0b0b0b0b0b0b0b0204020b0b0b050b0a" + - "0c0c02010d0d0d0d0d00060b0d0d0d0d020201020d0d0d0d0c0b02020d0d0d07" + - "04040404070606020b050402060602000e020202060205040404060606040202" + - "040404040404040404040404000000000f0f00090f0f0f0f0f0f0f0b09030d0d" + - "0909060909090101010100000909090909090909010101010101010101010101" + - "0101010101010101010101010d0d0d0d0d0d0d10090c0909090909100f0f0f0f" + - "0f0f07070f0f0f0f0f0f0e0e0f0f0f0f0f0f0f0f0c0c0c0c0c0c0c0c0c0c0c0c" + - "0c0c00080d0d0d0d0d0d020b0d0d0d0d030200010d0d0d0d0d0b02040d0d0d07" + - "0202020207060b020202020206060b0e0e040006060208080808080806060606" + - "00000000000000000000000009090909111111110d0d0d0d1212120900000000" + - "000107060a0a0505050500001212121212121212090909090909090909090909" + - "050511040d0d0d0d0d02040b0d070d0d0a0200010d0d0d0d050b02020d0d0d0d" + - "07020202060b0b0b0402050b02050b07010b00050202020202020c0002020202" + - "02020202020202020202020202020202020202020d0d0d0d0d0d0d0d09090909" + - "09090f0912121212121212121210101010101010090909090909090909090909" + - "0909090909090909090909090909090909090909090909090909090909090909" + - "090909090e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e" + - "0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e" + - "0e0e020211111111111111111111080808080808111111111111111111111111" + - "1111111111111111111111111111111111111111111111111111111111111111" + - "111111110e0e0e0e0e0e0e0e0e030303030303030e0e0e0e0e0e0e0e0e0e0e0e" + - "0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e0e" + - "0e0e0e0e03030303030303030303030303030303030303030303030303030303", - output: "0000010203000000040400050602070301010607030205000807070101040101" + - "04020201000002010908040300060003000a0400040400000000000905020000" + - "0804030501010100000000000b02020403020000000000010902000000000000" + - "0000000000000000000000000000000004010000000000000000000100010000" + - "0401020004000000010100000000000100010000000000010001000000000000" + - "0000000000000001000100000000000000000000000000010301020000030108" + - "0c0004090d00000000090907030000000500050102000000060403000300000c" + - "0a00000001070004050a0503040001090e020000030105050000030000010300" + - "010000000000000000000000050000000f00010e0200000000000008020f0b00" + - "0200080100000d00000007000200000000000000020000000000000000000000" + - "0000000000000000000000000400000000000010030e01000000000209000000" + - "00000e000100000000000e000100000000000000050000000000000000000000" + - "000008100800000000000e0d020000000d03050c040000000005040e0300000b" + - "03000000010e0503000000000200020c0006080400050a000000000002000000" + - "0300000000000000000000000e000000110000000a0000001200000304000000" + - "000c0c0712001200000005000700000000000000070000000000000000000000" + - "0300090c0a000000000c020e030b01000a050a0c040000000907050003000000" + - "070200000c040000090306030202020507020804050000000000100302000000" + - "000000000000000000000000000000000000000009000000000000000c000000" + - "000011010e000000000000000012000000000000020000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000011000000000000000000000000000000000000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000060010000000000000000000110000000000010000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000003000000000000000012000000000000010000000000000000000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000001000000000000000000000000000000000000000000000000000000", - }, { - input: "0000000000010101010102020202030303030404040405050505050606060707" + - "07070808", - output: "0000000000010000000002000000030000000400000005000000000600000700" + - "00000800", - }, { - input: "0000000000000000010001010101010101000000000000000002020303030303" + - "0303030303030303030303030303030303030303030303030303030303030303" + - "0303030303030303030401010101010101040005050505050502020303030303" + - "0303030303030303030303030303030303030303030303030303030303030303", - output: "0000000000000000010101000000000000010000000000000002000300000000" + - "0000000000000000000000000000000000000000000000000000000000000000" + - "0000000000000000000404000000000000010405000000000005000500000000" + - "0000000000000000000000000000000000000000000000000000000000000000", - }, { - input: "0000010000000102020101030303010404040105050501060606010707010108" + - "08010109", - output: "0000010100000102000100030000010400000105000001060000010700010008" + - "00010009", - }} - - var mtf moveToFront - for i, v := range vectors { - input, _ := hex.DecodeString(v.input) - mtf.Encode(input) - output := append([]uint8(nil), input...) - mtf.Decode(input) - - if input := hex.EncodeToString(input); input != v.input { - t.Errorf("test %d, input differs:\ngot %v\nwant %v", i, input, v.input) - } - if output := hex.EncodeToString(output); output != v.output { - t.Errorf("test %d, output differs:\ngot %v\nwant %v", i, output, v.output) - } - } -} - -// This package relies on dynamic generation of LUTs to reduce the static -// binary size. This benchmark attempts to measure the startup cost of init. -// This benchmark is not thread-safe; so do not run it in parallel with other -// tests or benchmarks! -func BenchmarkInit(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - initLUTs() - } -} diff --git a/vendor/github.com/dsnet/compress/brotli/dict_decoder_test.go b/vendor/github.com/dsnet/compress/brotli/dict_decoder_test.go deleted file mode 100644 index b2642cb8..00000000 --- a/vendor/github.com/dsnet/compress/brotli/dict_decoder_test.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli - -import "bytes" -import "strings" -import "testing" - -func TestDictDecoder(t *testing.T) { - const abc = "ABC\n" - const fox = "The quick brown fox jumped over the lazy dog!\n" - const poem = "The Road Not Taken\nRobert Frost\n" + - "\n" + - "Two roads diverged in a yellow wood,\n" + - "And sorry I could not travel both\n" + - "And be one traveler, long I stood\n" + - "And looked down one as far as I could\n" + - "To where it bent in the undergrowth;\n" + - "\n" + - "Then took the other, as just as fair,\n" + - "And having perhaps the better claim,\n" + - "Because it was grassy and wanted wear;\n" + - "Though as for that the passing there\n" + - "Had worn them really about the same,\n" + - "\n" + - "And both that morning equally lay\n" + - "In leaves no step had trodden black.\n" + - "Oh, I kept the first for another day!\n" + - "Yet knowing how way leads on to way,\n" + - "I doubted if I should ever come back.\n" + - "\n" + - "I shall be telling this with a sigh\n" + - "Somewhere ages and ages hence:\n" + - "Two roads diverged in a wood, and I-\n" + - "I took the one less traveled by,\n" + - "And that has made all the difference.\n" - var refs = []struct { - dist int // Backward distance (0 if this is an insertion) - length int // Length of copy or insertion - }{ - {0, 38}, {33, 3}, {0, 48}, {79, 3}, {0, 11}, {34, 5}, {0, 6}, {23, 7}, - {0, 8}, {50, 3}, {0, 2}, {69, 3}, {34, 5}, {0, 4}, {97, 3}, {0, 4}, - {43, 5}, {0, 6}, {7, 4}, {88, 7}, {0, 12}, {80, 3}, {0, 2}, {141, 4}, - {0, 1}, {196, 3}, {0, 3}, {157, 3}, {0, 6}, {181, 3}, {0, 2}, {23, 3}, - {77, 3}, {28, 5}, {128, 3}, {110, 4}, {70, 3}, {0, 4}, {85, 6}, {0, 2}, - {182, 6}, {0, 4}, {133, 3}, {0, 7}, {47, 5}, {0, 20}, {112, 5}, {0, 1}, - {58, 3}, {0, 8}, {59, 3}, {0, 4}, {173, 3}, {0, 5}, {114, 3}, {0, 4}, - {92, 5}, {0, 2}, {71, 3}, {0, 2}, {76, 5}, {0, 1}, {46, 3}, {96, 4}, - {130, 4}, {0, 3}, {360, 3}, {0, 3}, {178, 5}, {0, 7}, {75, 3}, {0, 3}, - {45, 6}, {0, 6}, {299, 6}, {180, 3}, {70, 6}, {0, 1}, {48, 3}, {66, 4}, - {0, 3}, {47, 5}, {0, 9}, {325, 3}, {0, 1}, {359, 3}, {318, 3}, {0, 2}, - {199, 3}, {0, 1}, {344, 3}, {0, 3}, {248, 3}, {0, 10}, {310, 3}, {0, 3}, - {93, 6}, {0, 3}, {252, 3}, {157, 4}, {0, 2}, {273, 5}, {0, 14}, {99, 4}, - {0, 1}, {464, 4}, {0, 2}, {92, 4}, {495, 3}, {0, 1}, {322, 4}, {16, 4}, - {0, 3}, {402, 3}, {0, 2}, {237, 4}, {0, 2}, {432, 4}, {0, 1}, {483, 5}, - {0, 2}, {294, 4}, {0, 2}, {306, 3}, {113, 5}, {0, 1}, {26, 4}, {164, 3}, - {488, 4}, {0, 1}, {542, 3}, {248, 6}, {0, 5}, {205, 3}, {0, 8}, {48, 3}, - {449, 6}, {0, 2}, {192, 3}, {328, 4}, {9, 5}, {433, 3}, {0, 3}, {622, 25}, - {615, 5}, {46, 5}, {0, 2}, {104, 3}, {475, 10}, {549, 3}, {0, 4}, {597, 8}, - {314, 3}, {0, 1}, {473, 6}, {317, 5}, {0, 1}, {400, 3}, {0, 3}, {109, 3}, - {151, 3}, {48, 4}, {0, 4}, {125, 3}, {108, 3}, {0, 2}, - } - - var want string - var buf bytes.Buffer - var dd dictDecoder - dd.Init(1 << 11) - - var checkLastBytes = func(str string) { - if len(str) < 2 { - str = "\x00\x00" + str - } - str = str[len(str)-2:] - p1, p2 := dd.LastBytes() - got := string([]byte{p2, p1}) - if got != str { - t.Errorf("last bytes mismatch: got %q, want %q", got, str) - } - } - var writeCopy = func(dist, length int) { - if dist < length { - cnt := (dist + length - 1) / dist - want += strings.Repeat(want[len(want)-dist:], cnt)[:length] - } else { - want += want[len(want)-dist:][:length] - } - - for length > 0 { - length -= dd.WriteCopy(dist, length) - if dd.AvailSize() == 0 { - buf.Write(dd.ReadFlush()) - } - } - - checkLastBytes(want) - } - var writeString = func(str string) { - want += str - - for len(str) > 0 { - cnt := copy(dd.WriteSlice(), str) - str = str[cnt:] - dd.WriteMark(cnt) - if dd.AvailSize() == 0 { - buf.Write(dd.ReadFlush()) - } - } - - checkLastBytes(want) - } - - writeString("") - writeString(".") - str := poem - for _, ref := range refs { - if ref.dist == 0 { - writeString(str[:ref.length]) - } else { - writeCopy(ref.dist, ref.length) - } - str = str[ref.length:] - } - writeCopy(dd.HistSize(), 33) - writeString(abc) - writeCopy(len(abc), 59*len(abc)) - writeString(fox) - writeCopy(len(fox), 9*len(fox)) - writeString(".") - writeCopy(1, 9) - writeString(strings.ToUpper(poem)) - writeCopy(len(poem), 7*len(poem)) - writeCopy(dd.HistSize(), 10) - - buf.Write(dd.ReadFlush()) - if buf.String() != want { - t.Errorf("final string mismatch:\ngot %q\nwant %q", buf.String(), want) - } -} - -func BenchmarkDictDecoderCopy(b *testing.B) { - nb := 1 << 24 - b.SetBytes(int64(nb)) - - for i := 0; i < b.N; i++ { - var dd dictDecoder - dd.Init(1 << 16) - - copy(dd.WriteSlice(), "abc") - dd.WriteMark(3) - - dist, length := 3, nb - for length > 0 { - length -= dd.WriteCopy(dist, length) - if dd.AvailSize() == 0 { - dd.ReadFlush() - } - } - } -} diff --git a/vendor/github.com/dsnet/compress/brotli/dict_encoder_test.go b/vendor/github.com/dsnet/compress/brotli/dict_encoder_test.go deleted file mode 100644 index 511a58fe..00000000 --- a/vendor/github.com/dsnet/compress/brotli/dict_encoder_test.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli diff --git a/vendor/github.com/dsnet/compress/brotli/prefix_test.go b/vendor/github.com/dsnet/compress/brotli/prefix_test.go deleted file mode 100644 index 511a58fe..00000000 --- a/vendor/github.com/dsnet/compress/brotli/prefix_test.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli diff --git a/vendor/github.com/dsnet/compress/brotli/reader_test.go b/vendor/github.com/dsnet/compress/brotli/reader_test.go deleted file mode 100644 index 4c28c016..00000000 --- a/vendor/github.com/dsnet/compress/brotli/reader_test.go +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli - -import "io" -import "io/ioutil" -import "bufio" -import "bytes" -import "strings" -import "encoding/hex" -import "runtime" -import "testing" - -func TestReader(t *testing.T) { - var vectors = []struct { - desc string // Description of the test - input string // Test input string in hex - output string // Expected output string in hex - inIdx int64 // Expected input offset after reading - outIdx int64 // Expected output offset after reading - err error // Expected error - }{{ - desc: "empty string (truncated)", - err: io.ErrUnexpectedEOF, - }, { - desc: "empty last block (WBITS: 16)", - input: "06", - inIdx: 1, - }, { - desc: "empty last block (WBITS: 12)", - input: "c101", - inIdx: 2, - }, { - desc: "empty last block (WBITS: 17)", - input: "8101", - inIdx: 2, - }, { - desc: "empty last block (WBITS: 21)", - input: "39", - inIdx: 1, - }, { - desc: "empty last block (WBITS: invalid)", - input: "9101", - inIdx: 1, - err: ErrCorrupt, - }, { - desc: "empty last block (trash at the end)", - input: "06ff", - inIdx: 1, - }, { - desc: "empty last block (padding is non-zero)", - input: "16", - inIdx: 1, - err: ErrCorrupt, - }, { - desc: "empty meta data block (MLEN: 0)", - input: "0c03", - inIdx: 2, - }, { - desc: "meta data block", - input: "2c0648656c6c6f2c20776f726c642103", - inIdx: 16, - }, { - desc: "meta data block (truncated)", - input: "2c06", - inIdx: 2, - err: io.ErrUnexpectedEOF, - }, { - desc: "meta data block (use reserved bit)", - input: "3c0648656c6c6f2c20776f726c642103", - inIdx: 1, - err: ErrCorrupt, - }, { - desc: "meta data block (meta padding is non-zero)", - input: "2c8648656c6c6f2c20776f726c642103", - inIdx: 2, - err: ErrCorrupt, - }, { - desc: "meta data block (non-minimal MLEN)", - input: "4c060048656c6c6f2c20776f726c642103", - inIdx: 3, - err: ErrCorrupt, - }, { - desc: "meta data block (MLEN: 1<<0)", - input: "2c00ff03", - inIdx: 4, - }, { - desc: "meta data block (MLEN: 1<<24)", - input: "ecffff7f" + strings.Repeat("f0", 1<<24) + "03", - inIdx: 5 + 1<<24, - }, { - desc: "raw data block", - input: "c0001048656c6c6f2c20776f726c642103", - output: "48656c6c6f2c20776f726c6421", - inIdx: 17, - outIdx: 13, - }, { - desc: "raw data block (truncated)", - input: "c00010", - inIdx: 3, - err: io.ErrUnexpectedEOF, - }, { - desc: "raw data block (raw padding is non-zero)", - input: "c000f048656c6c6f2c20776f726c642103", - inIdx: 3, - err: ErrCorrupt, - }, { - desc: "raw data block (non-minimal MLEN)", - input: "c400000148656c6c6f2c20776f726c642103", - inIdx: 3, - err: ErrCorrupt, - }, { - desc: "raw data block (MLEN: 1<<0)", - input: "0000106103", - output: "61", - inIdx: 4 + 1<<0, - outIdx: 1 << 0, - }, { - desc: "raw data block (MLEN: 1<<24)", - input: "f8ffff1f" + strings.Repeat("f0", 1<<24) + "03", - output: strings.Repeat("f0", 1<<24), - inIdx: 5 + 1<<24, - outIdx: 1 << 24, - }, { - desc: "simple prefix (|L|:1 |I|:1 |D|:1 MLEN:1)", - input: "00000000c4682010c0", - output: "a3", - inIdx: 9, - outIdx: 1, - }, { - desc: "simple prefix, out-of-order (|L|:2 |I|:1 |D|:1 MLEN:1)", - input: "00000000d4a8682010c001", - output: "a3", - inIdx: 11, - outIdx: 1, - }, { - desc: "simple prefix, non-unique (|L|:2 |I|:1 |D|:1 MLEN:1)", - input: "00000000d4e8682010c001", - output: "", - inIdx: 7, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "simple prefix, out-of-order (|L|:3 |I|:1 |D|:1 MLEN:1)", - input: "0000000024e8e96820104003", - output: "a3", - inIdx: 12, - outIdx: 1, - }, { - desc: "simple prefix, out-of-order, no-tree-select (|L|:4 |I|:1 |D|:1 MLEN:1)", - input: "0000000034e8e968a840208006", - output: "a3", - inIdx: 13, - outIdx: 1, - }, { - desc: "simple prefix, out-of-order, yes-tree-select (|L|:4 |I|:1 |D|:1 MLEN:1)", - input: "0000000034e8e968e94020800d", - output: "a3", - inIdx: 13, - outIdx: 1, - }, { - desc: "simple prefix, max-sym-ok (|L|:1 |I|:2 |D|:1 MLEN:1)", - input: "00000000c46821f06b0006", - output: "a3", - inIdx: 11, - outIdx: 1, - }, { - desc: "simple prefix, max-sym-bad (|L|:1 |I|:2 |D|:1 MLEN:1)", - input: "00000000c46821006c0006", - output: "", - inIdx: 9, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "complex prefix, skip-zero, terminate-clens-codes (|L|:1 |I|:2 |D|:1 MLEN:1)", - input: "0000000070472010c001", - output: "01", - inIdx: 10, - outIdx: 1, - }, { - desc: "complex prefix, skip-zero, terminate-clens-codes (|L|:1 |I|:2 |D|:1 MLEN:1)", - input: "0000000070c01d080470", - output: "01", - inIdx: 10, - outIdx: 1, - }, { - desc: "complex prefix, skip-zero, terminate-clens-codes (|L|:1 |I|:2 |D|:1 MLEN:2)", - input: "1000000070c01d1004d0", - output: "0100", - inIdx: 10, - outIdx: 2, - }, { - desc: "complex prefix, skip-zero, terminate-codes (|L|:1 |I|:4 |D|:1 MLEN:3)", - input: "20000000b0c100000056151804700e", - output: "030201", - inIdx: 15, - outIdx: 3, - }, { - desc: "complex prefix, skip-zero, under-subscribed (|L|:1 |I|:4 |D|:1 MLEN:3)", - input: "20000000b0c1000000ae2a3008e01c", - output: "", - inIdx: 10, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "complex prefix, skip-zero, over-subscribed (|L|:1 |I|:4 |D|:1 MLEN:3)", - input: "20000000b0c1000000ac0a0c023807", - output: "", - inIdx: 10, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "complex prefix, skip-zero, single clens (|L|:1 |I|:256 |D|:1 MLEN:4)", - input: "30000000000000020001420000a5ff5503", - output: "00a5ffaa", - inIdx: 17, - outIdx: 4, - }, { - desc: "complex prefix, skip-zero, single clens (|L|:1 |I|:32 |D|:1 MLEN:4)", - input: "3000000000c001000004080100faf7", - output: "00051f1b", - inIdx: 15, - outIdx: 4, - }, { - desc: "complex prefix, skip-zero, single clens, zero clen (|L|:1 |I|:? |D|:1 MLEN:4)", - input: "30000000007000000004080100faf7", - output: "", - inIdx: 10, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "complex prefix, skip-zero, empty clens (|L|:1 |I|:? |D|:1 MLEN:4)", - input: "30000000000000000001420080fe3d", - output: "", - inIdx: 9, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "complex prefix, skip-zero, single clens, rep-last clen (|L|:1 |I|:256 |D|:1 MLEN:4)", - input: "3000000000002000006a014200aa33cc5503", - output: "55cc33aa", - inIdx: 18, - outIdx: 4, - }, { - desc: "complex prefix, skip-zero, single clens, rep-last clen, over-subscribed (|L|:1 |I|:257 |D|:1 MLEN:4)", - input: "300000000000200000aa014200aa33cc5503", - output: "", - inIdx: 10, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "complex prefix, skip-zero, single clens, rep-last clen, integer overflow (|L|:1 |I|:1018 |D|:1 MLEN:4)", - input: "3000000000002000002a070801a8ce30570d", - output: "", - inIdx: 11, - outIdx: 0, - err: ErrCorrupt, - }, { - desc: "complex prefix, skip-two, single clens, rep-last clen (|L|:1 |I|:256 |D|:1 MLEN:4)", - input: "3000000008000f00805a801080ea0c73d5", - output: "55cc33aa", - inIdx: 17, - outIdx: 4, - }, { - desc: "complex prefix, skip-three, single clens, rep-last clen (|L|:1 |I|:256 |D|:1 MLEN:4)", - input: "300000000cc00300a0162004a03ac35c35", - output: "55cc33aa", - inIdx: 17, - outIdx: 4, - }, { - desc: "complex prefix, skip-zero, linear clens (|L|:1 |I|:16 |D|:1 MLEN:16)", - input: "f000000050555555ffff8bd5169058d43cb2fadcf77f201480dabdeff7f7efbf" + - "fffddffffbfffe7fffff01", - output: "6162636465666768696a6b6c6d6e6f70", - inIdx: 43, - outIdx: 16, - }, { - desc: "complex prefix, skip-zero, mixed clens (|L|:1 |I|:192 |D|:1 MLEN:16)", - input: "f000000050555555ffffe37a310f369a4d4b80756cc779b0619a02a1002c29ab" + - "ec066084eee99dfd67d8ac18", - output: "000240525356575e717a8abcbdbed7d9", - inIdx: 44, - outIdx: 16, - }, { - desc: "compressed string: \"Hello, world! Hello, world!\"", - input: "1b1a00008c946ed6540dc2825426d942de6a9668ea996c961e00", - output: "48656c6c6f2c20776f726c64212048656c6c6f2c20776f726c6421", - inIdx: 26, - outIdx: 27, - }, { - desc: "compressed string (padding is non-zero): \"Hello, world! Hello, world!\"", - input: "1b1a00008c946ed6540dc2825426d942de6a9668ea996c961e80", - output: "48656c6c6f2c20776f726c64212048656c6c6f2c20776f726c6421", - inIdx: 26, - outIdx: 27, - err: ErrCorrupt, - }} - - for i, v := range vectors { - input, _ := hex.DecodeString(v.input) - rd, err := NewReader(bytes.NewReader(input), nil) - if err != nil { - t.Errorf("test %d, unexpected NewReader error: %v", i, err) - } - data, err := ioutil.ReadAll(rd) - output := hex.EncodeToString(data) - - if err != v.err { - t.Errorf("test %d, %s\nerror mismatch: got %v, want %v", i, v.desc, err, v.err) - } - if output != v.output { - t.Errorf("test %d, %s\noutput mismatch:\ngot %v\nwant %v", i, v.desc, output, v.output) - } - if rd.InputOffset != v.inIdx { - t.Errorf("test %d, %s\ninput offset mismatch: got %d, want %d", i, v.desc, rd.InputOffset, v.inIdx) - } - if rd.OutputOffset != v.outIdx { - t.Errorf("test %d, %s\noutput offset mismatch: got %d, want %d", i, v.desc, rd.OutputOffset, v.outIdx) - } - } -} - -func TestReaderGolden(t *testing.T) { - var vectors = []struct { - input string // Input filename - output string // Output filename - }{ - {"empty.br", "empty"}, - {"empty.00.br", "empty"}, - {"empty.01.br", "empty"}, - {"empty.02.br", "empty"}, - {"empty.03.br", "empty"}, - {"empty.04.br", "empty"}, - {"empty.05.br", "empty"}, - {"empty.06.br", "empty"}, - {"empty.07.br", "empty"}, - {"empty.08.br", "empty"}, - {"empty.09.br", "empty"}, - {"empty.10.br", "empty"}, - {"empty.11.br", "empty"}, - {"empty.12.br", "empty"}, - {"empty.13.br", "empty"}, - {"empty.14.br", "empty"}, - {"empty.15.br", "empty"}, - {"empty.16.br", "empty"}, - {"empty.17.br", "empty"}, - {"empty.18.br", "empty"}, - {"zeros.br", "zeros"}, - {"x.br", "x"}, - {"x.00.br", "x"}, - {"x.01.br", "x"}, - {"x.02.br", "x"}, - {"x.03.br", "x"}, - {"xyzzy.br", "xyzzy"}, - {"10x10y.br", "10x10y"}, - {"64x.br", "64x"}, - {"backward65536.br", "backward65536"}, - {"quickfox.br", "quickfox"}, - {"quickfox_repeated.br", "quickfox_repeated"}, - {"ukkonooa.br", "ukkonooa"}, - {"monkey.br", "monkey"}, - {"random_org_10k.bin.br", "random_org_10k.bin"}, - {"asyoulik.txt.br", "asyoulik.txt"}, - {"compressed_file.br", "compressed_file"}, - {"compressed_repeated.br", "compressed_repeated"}, - {"alice29.txt.br", "alice29.txt"}, - {"lcet10.txt.br", "lcet10.txt"}, - {"mapsdatazrh.br", "mapsdatazrh"}, - {"plrabn12.txt.br", "plrabn12.txt"}, - } - - for i, v := range vectors { - input, err := ioutil.ReadFile("testdata/" + v.input) - if err != nil { - t.Errorf("test %d: %s\n%v", i, v.input, err) - continue - } - output, err := ioutil.ReadFile("testdata/" + v.output) - if err != nil { - t.Errorf("test %d: %s\n%v", i, v.output, err) - continue - } - - rd, err := NewReader(bytes.NewReader(input), nil) - if err != nil { - t.Errorf("test %d, unexpected NewReader error: %v", i, err) - } - data, err := ioutil.ReadAll(rd) - if err != nil { - t.Errorf("test %d, %s\nerror mismatch: got %v, want nil", i, v.input, err) - } - if string(data) != string(output) { - t.Errorf("test %d, %s\noutput mismatch:\ngot %q\nwant %q", i, v.input, string(data), string(output)) - } - } -} - -func benchmarkDecode(b *testing.B, testfile string) { - b.StopTimer() - b.ReportAllocs() - - input, err := ioutil.ReadFile("testdata/" + testfile) - if err != nil { - b.Fatal(err) - } - r, err := NewReader(bytes.NewReader(input), nil) - if err != nil { - b.Fatal(err) - } - output, err := ioutil.ReadAll(r) - if err != nil { - b.Fatal(err) - } - - nb := int64(len(output)) - output = nil - runtime.GC() - - b.SetBytes(nb) - b.StartTimer() - for i := 0; i < b.N; i++ { - r, err := NewReader(bufio.NewReader(bytes.NewReader(input)), nil) - if err != nil { - b.Fatal(err) - } - cnt, err := io.Copy(ioutil.Discard, r) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if cnt != nb { - b.Fatalf("unexpected count: got %d, want %d", cnt, nb) - } - } -} - -func BenchmarkDecodeDigitsSpeed1e4(b *testing.B) { benchmarkDecode(b, "digits-speed-1e4.br") } -func BenchmarkDecodeDigitsSpeed1e5(b *testing.B) { benchmarkDecode(b, "digits-speed-1e5.br") } -func BenchmarkDecodeDigitsSpeed1e6(b *testing.B) { benchmarkDecode(b, "digits-speed-1e6.br") } -func BenchmarkDecodeDigitsDefault1e4(b *testing.B) { benchmarkDecode(b, "digits-default-1e4.br") } -func BenchmarkDecodeDigitsDefault1e5(b *testing.B) { benchmarkDecode(b, "digits-default-1e5.br") } -func BenchmarkDecodeDigitsDefault1e6(b *testing.B) { benchmarkDecode(b, "digits-default-1e6.br") } -func BenchmarkDecodeDigitsCompress1e4(b *testing.B) { benchmarkDecode(b, "digits-best-1e4.br") } -func BenchmarkDecodeDigitsCompress1e5(b *testing.B) { benchmarkDecode(b, "digits-best-1e5.br") } -func BenchmarkDecodeDigitsCompress1e6(b *testing.B) { benchmarkDecode(b, "digits-best-1e6.br") } -func BenchmarkDecodeTwainSpeed1e4(b *testing.B) { benchmarkDecode(b, "twain-speed-1e4.br") } -func BenchmarkDecodeTwainSpeed1e5(b *testing.B) { benchmarkDecode(b, "twain-speed-1e5.br") } -func BenchmarkDecodeTwainSpeed1e6(b *testing.B) { benchmarkDecode(b, "twain-speed-1e6.br") } -func BenchmarkDecodeTwainDefault1e4(b *testing.B) { benchmarkDecode(b, "twain-default-1e4.br") } -func BenchmarkDecodeTwainDefault1e5(b *testing.B) { benchmarkDecode(b, "twain-default-1e5.br") } -func BenchmarkDecodeTwainDefault1e6(b *testing.B) { benchmarkDecode(b, "twain-default-1e6.br") } -func BenchmarkDecodeTwainCompress1e4(b *testing.B) { benchmarkDecode(b, "twain-best-1e4.br") } -func BenchmarkDecodeTwainCompress1e5(b *testing.B) { benchmarkDecode(b, "twain-best-1e5.br") } -func BenchmarkDecodeTwainCompress1e6(b *testing.B) { benchmarkDecode(b, "twain-best-1e6.br") } diff --git a/vendor/github.com/dsnet/compress/brotli/transform_test.go b/vendor/github.com/dsnet/compress/brotli/transform_test.go deleted file mode 100644 index b0265f37..00000000 --- a/vendor/github.com/dsnet/compress/brotli/transform_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli - -import "testing" - -func TestTransform(t *testing.T) { - var vectors = []struct { - id int - input string - output string - }{ - {id: 0, input: "Hello, world!", output: "Hello, world!"}, - {id: 23, input: "groups of", output: "groups"}, - {id: 42, input: "s for the ", output: "s for "}, - {id: 48, input: "presentation", output: "prese"}, - {id: 56, input: "maintenance", output: "maint"}, - {id: 23, input: "Alexandria", output: "Alexand"}, - {id: 23, input: "archives", output: "archi"}, - {id: 49, input: "fighty", output: "fighting "}, - {id: 49, input: "12", output: "1ing "}, - {id: 49, input: "1", output: "ing "}, - {id: 49, input: "", output: "ing "}, - {id: 64, input: "123456789a", output: "1"}, - {id: 64, input: "123456789", output: ""}, - {id: 64, input: "1", output: ""}, - {id: 64, input: "", output: ""}, - {id: 3, input: "afloat", output: "float"}, - {id: 3, input: "12", output: "2"}, - {id: 3, input: "1", output: ""}, - {id: 3, input: "", output: ""}, - {id: 54, input: "123456789a", output: "a"}, - {id: 54, input: "123456789", output: ""}, - {id: 54, input: "1", output: ""}, - {id: 54, input: "", output: ""}, - {id: 73, input: "", output: " the of the "}, - {id: 73, input: "dichlorodifluoromethanes", output: " the dichlorodifluoromethanes of the "}, - {id: 15, input: "", output: " "}, - {id: 15, input: "meow", output: " Meow "}, - {id: 15, input: "-scale", output: " -scale "}, - {id: 15, input: "почти", output: " Почти "}, - {id: 15, input: "互联网", output: " 亗联网 "}, - {id: 119, input: "", output: " ='"}, - {id: 119, input: "meow", output: " MEOW='"}, - {id: 119, input: "-scale", output: " -SCALE='"}, - {id: 119, input: "почти", output: " ПОѧѢИ='"}, - {id: 119, input: "互联网", output: " 亗聑罔='"}, - } - - var buf [maxWordSize]byte - for i, v := range vectors { - cnt := transformWord(buf[:], []byte(v.input), v.id) - output := string(buf[:cnt]) - - if output != v.output { - t.Errorf("test %d, output mismatch: got %q, want %q", i, output, v.output) - } - } -} diff --git a/vendor/github.com/dsnet/compress/brotli/writer_test.go b/vendor/github.com/dsnet/compress/brotli/writer_test.go deleted file mode 100644 index 511a58fe..00000000 --- a/vendor/github.com/dsnet/compress/brotli/writer_test.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2015, Joe Tsai. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE.md file. - -package brotli diff --git a/vendor/gopkg.in/fsnotify.v1/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS similarity index 85% rename from vendor/gopkg.in/fsnotify.v1/AUTHORS rename to vendor/github.com/fsnotify/fsnotify/AUTHORS index 2b9d1fdd..0a5bf8f6 100644 --- a/vendor/gopkg.in/fsnotify.v1/AUTHORS +++ b/vendor/github.com/fsnotify/fsnotify/AUTHORS @@ -9,6 +9,9 @@ # Please keep the list sorted. Adrien Bustany +Amit Krishnan +Bjørn Erik Pedersen +Bruno Bigras Caleb Spare Case Nelson Chris Howey @@ -23,13 +26,16 @@ Kelvin Fo Ken-ichirou MATSUZAWA Matt Layher Nathan Youngman +Patrick Paul Hammond Pawel Knap Pieter Droogendijk Pursuit92 Riku Voipio Rob Figueiredo +Slawek Ligus Soge Zhang +Tiffany Jernigan Tilak Sharma Travis Cline Tudor Golubenco diff --git a/vendor/gopkg.in/fsnotify.v1/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md similarity index 74% rename from vendor/gopkg.in/fsnotify.v1/CHANGELOG.md rename to vendor/github.com/fsnotify/fsnotify/CHANGELOG.md index d37247e0..8c732c1d 100644 --- a/vendor/gopkg.in/fsnotify.v1/CHANGELOG.md +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -1,72 +1,96 @@ # Changelog +## v1.4.2 / 2016-10-10 + +* Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) + +## v1.4.1 / 2016-10-04 + +* Fix flaky inotify stress test on Linux [#177](https://github.com/fsnotify/fsnotify/pull/177) (thanks @pattyshack) + +## v1.4.0 / 2016-10-01 + +* add a String() method to Event.Op [#165](https://github.com/fsnotify/fsnotify/pull/165) (thanks @oozie) + +## v1.3.1 / 2016-06-28 + +* Windows: fix for double backslash when watching the root of a drive [#151](https://github.com/fsnotify/fsnotify/issues/151) (thanks @brunoqc) + +## v1.3.0 / 2016-04-19 + +* Support linux/arm64 by [patching](https://go-review.googlesource.com/#/c/21971/) x/sys/unix and switching to to it from syscall (thanks @suihkulokki) [#135](https://github.com/fsnotify/fsnotify/pull/135) + +## v1.2.10 / 2016-03-02 + +* Fix golint errors in windows.go [#121](https://github.com/fsnotify/fsnotify/pull/121) (thanks @tiffanyfj) + ## v1.2.9 / 2016-01-13 -kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/go-fsnotify/fsnotify/pull/111) (thanks @bep) +kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsnotify/pull/111) (thanks @bep) ## v1.2.8 / 2015-12-17 -* kqueue: fix race condition in Close [#105](https://github.com/go-fsnotify/fsnotify/pull/105) (thanks @djui for reporting the issue and @ppknap for writing a failing test) +* kqueue: fix race condition in Close [#105](https://github.com/fsnotify/fsnotify/pull/105) (thanks @djui for reporting the issue and @ppknap for writing a failing test) * inotify: fix race in test * enable race detection for continuous integration (Linux, Mac, Windows) ## v1.2.5 / 2015-10-17 -* inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/go-fsnotify/fsnotify/pull/100) (thanks @suihkulokki) -* inotify: fix path leaks [#73](https://github.com/go-fsnotify/fsnotify/pull/73) (thanks @chamaken) -* kqueue: watch for rename events on subdirectories [#83](https://github.com/go-fsnotify/fsnotify/pull/83) (thanks @guotie) -* kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/go-fsnotify/fsnotify/pull/101) (thanks @illicitonion) +* inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/fsnotify/fsnotify/pull/100) (thanks @suihkulokki) +* inotify: fix path leaks [#73](https://github.com/fsnotify/fsnotify/pull/73) (thanks @chamaken) +* kqueue: watch for rename events on subdirectories [#83](https://github.com/fsnotify/fsnotify/pull/83) (thanks @guotie) +* kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/fsnotify/fsnotify/pull/101) (thanks @illicitonion) ## v1.2.1 / 2015-10-14 -* kqueue: don't watch named pipes [#98](https://github.com/go-fsnotify/fsnotify/pull/98) (thanks @evanphx) +* kqueue: don't watch named pipes [#98](https://github.com/fsnotify/fsnotify/pull/98) (thanks @evanphx) ## v1.2.0 / 2015-02-08 -* inotify: use epoll to wake up readEvents [#66](https://github.com/go-fsnotify/fsnotify/pull/66) (thanks @PieterD) -* inotify: closing watcher should now always shut down goroutine [#63](https://github.com/go-fsnotify/fsnotify/pull/63) (thanks @PieterD) -* kqueue: close kqueue after removing watches, fixes [#59](https://github.com/go-fsnotify/fsnotify/issues/59) +* inotify: use epoll to wake up readEvents [#66](https://github.com/fsnotify/fsnotify/pull/66) (thanks @PieterD) +* inotify: closing watcher should now always shut down goroutine [#63](https://github.com/fsnotify/fsnotify/pull/63) (thanks @PieterD) +* kqueue: close kqueue after removing watches, fixes [#59](https://github.com/fsnotify/fsnotify/issues/59) ## v1.1.1 / 2015-02-05 -* inotify: Retry read on EINTR [#61](https://github.com/go-fsnotify/fsnotify/issues/61) (thanks @PieterD) +* inotify: Retry read on EINTR [#61](https://github.com/fsnotify/fsnotify/issues/61) (thanks @PieterD) ## v1.1.0 / 2014-12-12 -* kqueue: rework internals [#43](https://github.com/go-fsnotify/fsnotify/pull/43) +* kqueue: rework internals [#43](https://github.com/fsnotify/fsnotify/pull/43) * add low-level functions * only need to store flags on directories - * less mutexes [#13](https://github.com/go-fsnotify/fsnotify/issues/13) + * less mutexes [#13](https://github.com/fsnotify/fsnotify/issues/13) * done can be an unbuffered channel * remove calls to os.NewSyscallError -* More efficient string concatenation for Event.String() [#52](https://github.com/go-fsnotify/fsnotify/pull/52) (thanks @mdlayher) -* kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/go-fsnotify/fsnotify/issues/48) -* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/go-fsnotify/fsnotify/issues/51) +* More efficient string concatenation for Event.String() [#52](https://github.com/fsnotify/fsnotify/pull/52) (thanks @mdlayher) +* kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/fsnotify/fsnotify/issues/48) +* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) ## v1.0.4 / 2014-09-07 * kqueue: add dragonfly to the build tags. * Rename source code files, rearrange code so exported APIs are at the top. -* Add done channel to example code. [#37](https://github.com/go-fsnotify/fsnotify/pull/37) (thanks @chenyukang) +* Add done channel to example code. [#37](https://github.com/fsnotify/fsnotify/pull/37) (thanks @chenyukang) ## v1.0.3 / 2014-08-19 -* [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/go-fsnotify/fsnotify/issues/36) +* [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/fsnotify/fsnotify/issues/36) ## v1.0.2 / 2014-08-17 -* [Fix] Missing create events on OS X. [#14](https://github.com/go-fsnotify/fsnotify/issues/14) (thanks @zhsso) +* [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) * [Fix] Make ./path and path equivalent. (thanks @zhsso) ## v1.0.0 / 2014-08-15 * [API] Remove AddWatch on Windows, use Add. -* Improve documentation for exported identifiers. [#30](https://github.com/go-fsnotify/fsnotify/issues/30) +* Improve documentation for exported identifiers. [#30](https://github.com/fsnotify/fsnotify/issues/30) * Minor updates based on feedback from golint. ## dev / 2014-07-09 -* Moved to [github.com/go-fsnotify/fsnotify](https://github.com/go-fsnotify/fsnotify). +* Moved to [github.com/fsnotify/fsnotify](https://github.com/fsnotify/fsnotify). * Use os.NewSyscallError instead of returning errno (thanks @hariharan-uno) ## dev / 2014-07-04 @@ -76,7 +100,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/go-fsnotify/ ## dev / 2014-06-28 -* [API] Don't set the Write Op for attribute notifications [#4](https://github.com/go-fsnotify/fsnotify/issues/4) +* [API] Don't set the Write Op for attribute notifications [#4](https://github.com/fsnotify/fsnotify/issues/4) * Fix for String() method on Event (thanks Alex Brainman) * Don't build on Plan 9 or Solaris (thanks @4ad) @@ -114,11 +138,11 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/go-fsnotify/ ## v0.9.3 / 2014-12-31 -* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/go-fsnotify/fsnotify/issues/51) +* kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) ## v0.9.2 / 2014-08-17 -* [Backport] Fix missing create events on OS X. [#14](https://github.com/go-fsnotify/fsnotify/issues/14) (thanks @zhsso) +* [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) ## v0.9.1 / 2014-06-12 @@ -137,7 +161,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/go-fsnotify/ ## v0.8.11 / 2013-11-02 * [Doc] Add Changelog [#72][] (thanks @nathany) -* [Doc] Spotlight and double modify events on OS X [#62][] (reported by @paulhammond) +* [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond) ## v0.8.10 / 2013-10-19 diff --git a/vendor/gopkg.in/fsnotify.v1/CONTRIBUTING.md b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md similarity index 76% rename from vendor/gopkg.in/fsnotify.v1/CONTRIBUTING.md rename to vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md index 0f377f34..828a60b2 100644 --- a/vendor/gopkg.in/fsnotify.v1/CONTRIBUTING.md +++ b/vendor/github.com/fsnotify/fsnotify/CONTRIBUTING.md @@ -2,7 +2,7 @@ ## Issues -* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/go-fsnotify/fsnotify/issues). +* Request features and report bugs using the [GitHub Issue Tracker](https://github.com/fsnotify/fsnotify/issues). * Please indicate the platform you are using fsnotify on. * A code example to reproduce the problem is appreciated. @@ -10,14 +10,14 @@ ### Contributor License Agreement -fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/go-fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/go-fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). +fsnotify is derived from code in the [golang.org/x/exp](https://godoc.org/golang.org/x/exp) package and it may be included [in the standard library](https://github.com/fsnotify/fsnotify/issues/1) in the future. Therefore fsnotify carries the same [LICENSE](https://github.com/fsnotify/fsnotify/blob/master/LICENSE) as Go. Contributors retain their copyright, so you need to fill out a short form before we can accept your contribution: [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual). Please indicate that you have signed the CLA in your pull request. ### How fsnotify is Developed * Development is done on feature branches. -* Tests are run on BSD, Linux, OS X and Windows. +* Tests are run on BSD, Linux, macOS and Windows. * Pull requests are reviewed and [applied to master][am] using [hub][]. * Maintainers may modify or squash commits rather than asking contributors to. * To issue a new release, the maintainers will: @@ -28,7 +28,7 @@ Please indicate that you have signed the CLA in your pull request. For smooth sailing, always use the original import path. Installing with `go get` makes this easy. -1. Install from GitHub (`go get -u github.com/go-fsnotify/fsnotify`) +1. Install from GitHub (`go get -u github.com/fsnotify/fsnotify`) 2. Create your feature branch (`git checkout -b my-new-feature`) 3. Ensure everything works and the tests pass (see below) 4. Commit your changes (`git commit -am 'Add some feature'`) @@ -40,11 +40,11 @@ Contribute upstream: 3. Push to the branch (`git push fork my-new-feature`) 4. Create a new Pull Request on GitHub -This workflow is [thoroughly explained by Katrina Owen](https://blog.splice.com/contributing-open-source-git-repositories-go/). +This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/contributing-open-source-git-repositories-go/). ### Testing -fsnotify uses build tags to compile different code on Linux, BSD, OS X, and Windows. +fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows. Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. @@ -53,12 +53,12 @@ To aid in cross-platform testing there is a Vagrantfile for Linux and BSD. * Install [Vagrant](http://www.vagrantup.com/) and [VirtualBox](https://www.virtualbox.org/) * Setup [Vagrant Gopher](https://github.com/nathany/vagrant-gopher) in your `src` folder. * Run `vagrant up` from the project folder. You can also setup just one box with `vagrant up linux` or `vagrant up bsd` (note: the BSD box doesn't support Windows hosts at this time, and NFS may prompt for your host OS password) -* Once setup, you can run the test suite on a given OS with a single command `vagrant ssh linux -c 'cd go-fsnotify/fsnotify; go test'`. +* Once setup, you can run the test suite on a given OS with a single command `vagrant ssh linux -c 'cd fsnotify/fsnotify; go test'`. * When you're done, you will want to halt or destroy the Vagrant boxes. Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory. -Right now there is no equivalent solution for Windows and OS X, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). +Right now there is no equivalent solution for Windows and macOS, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). ### Maintainers diff --git a/vendor/gopkg.in/fsnotify.v1/LICENSE b/vendor/github.com/fsnotify/fsnotify/LICENSE similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/LICENSE rename to vendor/github.com/fsnotify/fsnotify/LICENSE diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md new file mode 100644 index 00000000..39932074 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -0,0 +1,79 @@ +# File system notifications for Go + +[![GoDoc](https://godoc.org/github.com/fsnotify/fsnotify?status.svg)](https://godoc.org/github.com/fsnotify/fsnotify) [![Go Report Card](https://goreportcard.com/badge/github.com/fsnotify/fsnotify)](https://goreportcard.com/report/github.com/fsnotify/fsnotify) + +fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather than `syscall` from the standard library. Ensure you have the latest version installed by running: + +```console +go get -u golang.org/x/sys/... +``` + +Cross platform: Windows, Linux, BSD and macOS. + +|Adapter |OS |Status | +|----------|----------|----------| +|inotify |Linux 2.6.27 or later, Android\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| +|kqueue |BSD, macOS, iOS\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| +|ReadDirectoryChangesW|Windows|Supported [![Build status](https://ci.appveyor.com/api/projects/status/ivwjubaih4r0udeh/branch/master?svg=true)](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)| +|FSEvents |macOS |[Planned](https://github.com/fsnotify/fsnotify/issues/11)| +|FEN |Solaris 11 |[In Progress](https://github.com/fsnotify/fsnotify/issues/12)| +|fanotify |Linux 2.6.37+ | | +|USN Journals |Windows |[Maybe](https://github.com/fsnotify/fsnotify/issues/53)| +|Polling |*All* |[Maybe](https://github.com/fsnotify/fsnotify/issues/9)| + +\* Android and iOS are untested. + +Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information. + +## API stability + +fsnotify is a fork of [howeyc/fsnotify](https://godoc.org/github.com/howeyc/fsnotify) with a new API as of v1.0. The API is based on [this design document](http://goo.gl/MrYxyA). + +All [releases](https://github.com/fsnotify/fsnotify/releases) are tagged based on [Semantic Versioning](http://semver.org/). Further API changes are [planned](https://github.com/fsnotify/fsnotify/milestones), and will be tagged with a new major revision number. + +Go 1.6 supports dependencies located in the `vendor/` folder. Unless you are creating a library, it is recommended that you copy fsnotify into `vendor/github.com/fsnotify/fsnotify` within your project, and likewise for `golang.org/x/sys`. + +## Contributing + +Please refer to [CONTRIBUTING][] before opening an issue or pull request. + +## Example + +See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go). + +## FAQ + +**When a file is moved to another directory is it still being watched?** + +No (it shouldn't be, unless you are watching where it was moved to). + +**When I watch a directory, are all subdirectories watched as well?** + +No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]). + +**Do I have to watch the Error and Event channels in a separate goroutine?** + +As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7]) + +**Why am I receiving multiple events for the same file on OS X?** + +Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]). + +**How many files can be watched at once?** + +There are OS-specific limits as to how many watches can be created: +* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. +* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. + +[#62]: https://github.com/howeyc/fsnotify/issues/62 +[#18]: https://github.com/fsnotify/fsnotify/issues/18 +[#11]: https://github.com/fsnotify/fsnotify/issues/11 +[#7]: https://github.com/howeyc/fsnotify/issues/7 + +[contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md + +## Related Projects + +* [notify](https://github.com/rjeczalik/notify) +* [fsevents](https://github.com/fsnotify/fsevents) + diff --git a/vendor/github.com/fsnotify/fsnotify/fen.go b/vendor/github.com/fsnotify/fsnotify/fen.go new file mode 100644 index 00000000..ced39cb8 --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/fen.go @@ -0,0 +1,37 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build solaris + +package fsnotify + +import ( + "errors" +) + +// Watcher watches a set of files, delivering events to a channel. +type Watcher struct { + Events chan Event + Errors chan error +} + +// NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. +func NewWatcher() (*Watcher, error) { + return nil, errors.New("FEN based watcher not yet supported for fsnotify\n") +} + +// Close removes all watches and closes the events channel. +func (w *Watcher) Close() error { + return nil +} + +// Add starts watching the named file or directory (non-recursively). +func (w *Watcher) Add(name string) error { + return nil +} + +// Remove stops watching the the named file or directory (non-recursively). +func (w *Watcher) Remove(name string) error { + return nil +} diff --git a/vendor/gopkg.in/fsnotify.v1/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go similarity index 72% rename from vendor/gopkg.in/fsnotify.v1/fsnotify.go rename to vendor/github.com/fsnotify/fsnotify/fsnotify.go index c899ee00..190bf0de 100644 --- a/vendor/gopkg.in/fsnotify.v1/fsnotify.go +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -2,13 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !plan9,!solaris +// +build !plan9 // Package fsnotify provides a platform-independent interface for file system notifications. package fsnotify import ( "bytes" + "errors" "fmt" ) @@ -30,33 +31,36 @@ const ( Chmod ) -// String returns a string representation of the event in the form -// "file: REMOVE|WRITE|..." -func (e Event) String() string { +func (op Op) String() string { // Use a buffer for efficient string concatenation var buffer bytes.Buffer - if e.Op&Create == Create { + if op&Create == Create { buffer.WriteString("|CREATE") } - if e.Op&Remove == Remove { + if op&Remove == Remove { buffer.WriteString("|REMOVE") } - if e.Op&Write == Write { + if op&Write == Write { buffer.WriteString("|WRITE") } - if e.Op&Rename == Rename { + if op&Rename == Rename { buffer.WriteString("|RENAME") } - if e.Op&Chmod == Chmod { + if op&Chmod == Chmod { buffer.WriteString("|CHMOD") } - - // If buffer remains empty, return no event names if buffer.Len() == 0 { - return fmt.Sprintf("%q: ", e.Name) + return "" } - - // Return a list of event names, with leading pipe character stripped - return fmt.Sprintf("%q: %s", e.Name, buffer.String()[1:]) + return buffer.String()[1:] // Strip leading pipe } + +// String returns a string representation of the event in the form +// "file: REMOVE|WRITE|..." +func (e Event) String() string { + return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) +} + +// Common errors that can be reported by a watcher +var ErrEventOverflow = errors.New("fsnotify queue overflow") diff --git a/vendor/gopkg.in/fsnotify.v1/inotify.go b/vendor/github.com/fsnotify/fsnotify/inotify.go similarity index 71% rename from vendor/gopkg.in/fsnotify.v1/inotify.go rename to vendor/github.com/fsnotify/fsnotify/inotify.go index 06f4bba8..d9fd1b88 100644 --- a/vendor/gopkg.in/fsnotify.v1/inotify.go +++ b/vendor/github.com/fsnotify/fsnotify/inotify.go @@ -14,8 +14,9 @@ import ( "path/filepath" "strings" "sync" - "syscall" "unsafe" + + "golang.org/x/sys/unix" ) // Watcher watches a set of files, delivering events to a channel. @@ -23,7 +24,6 @@ type Watcher struct { Events chan Event Errors chan error mu sync.Mutex // Map access - cv *sync.Cond // sync removing on rm_watch with IN_IGNORE fd int poller *fdPoller watches map[string]*watch // Map of inotify watches (key: path) @@ -35,14 +35,14 @@ type Watcher struct { // NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. func NewWatcher() (*Watcher, error) { // Create inotify fd - fd, errno := syscall.InotifyInit() + fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC) if fd == -1 { return nil, errno } // Create epoll poller, err := newFdPoller(fd) if err != nil { - syscall.Close(fd) + unix.Close(fd) return nil, err } w := &Watcher{ @@ -55,7 +55,6 @@ func NewWatcher() (*Watcher, error) { done: make(chan struct{}), doneResp: make(chan struct{}), } - w.cv = sync.NewCond(&w.mu) go w.readEvents() return w, nil @@ -95,28 +94,30 @@ func (w *Watcher) Add(name string) error { return errors.New("inotify instance already closed") } - const agnosticEvents = syscall.IN_MOVED_TO | syscall.IN_MOVED_FROM | - syscall.IN_CREATE | syscall.IN_ATTRIB | syscall.IN_MODIFY | - syscall.IN_MOVE_SELF | syscall.IN_DELETE | syscall.IN_DELETE_SELF + const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | + unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | + unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF var flags uint32 = agnosticEvents w.mu.Lock() - watchEntry, found := w.watches[name] - w.mu.Unlock() - if found { - watchEntry.flags |= flags - flags |= syscall.IN_MASK_ADD + defer w.mu.Unlock() + watchEntry := w.watches[name] + if watchEntry != nil { + flags |= watchEntry.flags | unix.IN_MASK_ADD } - wd, errno := syscall.InotifyAddWatch(w.fd, name, flags) + wd, errno := unix.InotifyAddWatch(w.fd, name, flags) if wd == -1 { return errno } - w.mu.Lock() - w.watches[name] = &watch{wd: uint32(wd), flags: flags} - w.paths[wd] = name - w.mu.Unlock() + if watchEntry == nil { + w.watches[name] = &watch{wd: uint32(wd), flags: flags} + w.paths[wd] = name + } else { + watchEntry.wd = uint32(wd) + watchEntry.flags = flags + } return nil } @@ -134,13 +135,20 @@ func (w *Watcher) Remove(name string) error { if !ok { return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) } + + // We successfully removed the watch if InotifyRmWatch doesn't return an + // error, we need to clean up our internal state to ensure it matches + // inotify's kernel state. + delete(w.paths, int(watch.wd)) + delete(w.watches, name) + // inotify_rm_watch will return EINVAL if the file has been deleted; // the inotify will already have been removed. // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously // by calling inotify_rm_watch() below. e.g. readEvents() goroutine receives IN_IGNORE // so that EINVAL means that the wd is being rm_watch()ed or its file removed // by another thread and we have not received IN_IGNORE event. - success, errno := syscall.InotifyRmWatch(w.fd, watch.wd) + success, errno := unix.InotifyRmWatch(w.fd, watch.wd) if success == -1 { // TODO: Perhaps it's not helpful to return an error here in every case. // the only two possible errors are: @@ -151,13 +159,6 @@ func (w *Watcher) Remove(name string) error { return errno } - // wait until ignoreLinux() deleting maps - exists := true - for exists { - w.cv.Wait() - _, exists = w.watches[name] - } - return nil } @@ -170,16 +171,16 @@ type watch struct { // received events into Event objects and sends them via the Events channel func (w *Watcher) readEvents() { var ( - buf [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events - n int // Number of bytes read with read() - errno error // Syscall errno - ok bool // For poller.wait + buf [unix.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events + n int // Number of bytes read with read() + errno error // Syscall errno + ok bool // For poller.wait ) defer close(w.doneResp) defer close(w.Errors) defer close(w.Events) - defer syscall.Close(w.fd) + defer unix.Close(w.fd) defer w.poller.close() for { @@ -202,26 +203,26 @@ func (w *Watcher) readEvents() { continue } - n, errno = syscall.Read(w.fd, buf[:]) + n, errno = unix.Read(w.fd, buf[:]) // If a signal interrupted execution, see if we've been asked to close, and try again. // http://man7.org/linux/man-pages/man7/signal.7.html : // "Before Linux 3.8, reads from an inotify(7) file descriptor were not restartable" - if errno == syscall.EINTR { + if errno == unix.EINTR { continue } - // syscall.Read might have been woken up by Close. If so, we're done. + // unix.Read might have been woken up by Close. If so, we're done. if w.isClosed() { return } - if n < syscall.SizeofInotifyEvent { + if n < unix.SizeofInotifyEvent { var err error if n == 0 { // If EOF is received. This should really never happen. err = io.EOF } else if n < 0 { - // If an error occured while reading. + // If an error occurred while reading. err = errno } else { // Read was too short. @@ -238,22 +239,40 @@ func (w *Watcher) readEvents() { var offset uint32 // We don't know how many events we just read into the buffer // While the offset points to at least one whole event... - for offset <= uint32(n-syscall.SizeofInotifyEvent) { + for offset <= uint32(n-unix.SizeofInotifyEvent) { // Point "raw" to the event in the buffer - raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset])) + raw := (*unix.InotifyEvent)(unsafe.Pointer(&buf[offset])) mask := uint32(raw.Mask) nameLen := uint32(raw.Len) + + if mask&unix.IN_Q_OVERFLOW != 0 { + select { + case w.Errors <- ErrEventOverflow: + case <-w.done: + return + } + } + // If the event happened to the watched directory or the watched file, the kernel // doesn't append the filename to the event, but we would like to always fill the // the "Name" field with a valid filename. We retrieve the path of the watch from // the "paths" map. w.mu.Lock() - name := w.paths[int(raw.Wd)] + name, ok := w.paths[int(raw.Wd)] + // IN_DELETE_SELF occurs when the file/directory being watched is removed. + // This is a sign to clean up the maps, otherwise we are no longer in sync + // with the inotify kernel state which has already deleted the watch + // automatically. + if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { + delete(w.paths, int(raw.Wd)) + delete(w.watches, name) + } w.mu.Unlock() + if nameLen > 0 { // Point "bytes" at the first byte of the filename - bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent])) + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent])) // The filename is padded with NULL bytes. TrimRight() gets rid of those. name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") } @@ -261,7 +280,7 @@ func (w *Watcher) readEvents() { event := newEvent(name, mask) // Send the events that are not ignored on the events channel - if !event.ignoreLinux(w, raw.Wd, mask) { + if !event.ignoreLinux(mask) { select { case w.Events <- event: case <-w.done: @@ -270,7 +289,7 @@ func (w *Watcher) readEvents() { } // Move to the next event in the buffer - offset += syscall.SizeofInotifyEvent + nameLen + offset += unix.SizeofInotifyEvent + nameLen } } } @@ -278,15 +297,9 @@ func (w *Watcher) readEvents() { // Certain types of events can be "ignored" and not sent over the Events // channel. Such as events marked ignore by the kernel, or MODIFY events // against files that do not exist. -func (e *Event) ignoreLinux(w *Watcher, wd int32, mask uint32) bool { +func (e *Event) ignoreLinux(mask uint32) bool { // Ignore anything the inotify API says to ignore - if mask&syscall.IN_IGNORED == syscall.IN_IGNORED { - w.mu.Lock() - defer w.mu.Unlock() - name := w.paths[int(wd)] - delete(w.paths, int(wd)) - delete(w.watches, name) - w.cv.Broadcast() + if mask&unix.IN_IGNORED == unix.IN_IGNORED { return true } @@ -305,19 +318,19 @@ func (e *Event) ignoreLinux(w *Watcher, wd int32, mask uint32) bool { // newEvent returns an platform-independent Event based on an inotify mask. func newEvent(name string, mask uint32) Event { e := Event{Name: name} - if mask&syscall.IN_CREATE == syscall.IN_CREATE || mask&syscall.IN_MOVED_TO == syscall.IN_MOVED_TO { + if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { e.Op |= Create } - if mask&syscall.IN_DELETE_SELF == syscall.IN_DELETE_SELF || mask&syscall.IN_DELETE == syscall.IN_DELETE { + if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { e.Op |= Remove } - if mask&syscall.IN_MODIFY == syscall.IN_MODIFY { + if mask&unix.IN_MODIFY == unix.IN_MODIFY { e.Op |= Write } - if mask&syscall.IN_MOVE_SELF == syscall.IN_MOVE_SELF || mask&syscall.IN_MOVED_FROM == syscall.IN_MOVED_FROM { + if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { e.Op |= Rename } - if mask&syscall.IN_ATTRIB == syscall.IN_ATTRIB { + if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { e.Op |= Chmod } return e diff --git a/vendor/gopkg.in/fsnotify.v1/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go similarity index 76% rename from vendor/gopkg.in/fsnotify.v1/inotify_poller.go rename to vendor/github.com/fsnotify/fsnotify/inotify_poller.go index 23a5ca14..cc7db4b2 100644 --- a/vendor/gopkg.in/fsnotify.v1/inotify_poller.go +++ b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go @@ -8,7 +8,8 @@ package fsnotify import ( "errors" - "syscall" + + "golang.org/x/sys/unix" ) type fdPoller struct { @@ -39,32 +40,32 @@ func newFdPoller(fd int) (*fdPoller, error) { poller.fd = fd // Create epoll fd - poller.epfd, errno = syscall.EpollCreate1(0) + poller.epfd, errno = unix.EpollCreate1(0) if poller.epfd == -1 { return nil, errno } // Create pipe; pipe[0] is the read end, pipe[1] the write end. - errno = syscall.Pipe2(poller.pipe[:], syscall.O_NONBLOCK) + errno = unix.Pipe2(poller.pipe[:], unix.O_NONBLOCK) if errno != nil { return nil, errno } // Register inotify fd with epoll - event := syscall.EpollEvent{ + event := unix.EpollEvent{ Fd: int32(poller.fd), - Events: syscall.EPOLLIN, + Events: unix.EPOLLIN, } - errno = syscall.EpollCtl(poller.epfd, syscall.EPOLL_CTL_ADD, poller.fd, &event) + errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.fd, &event) if errno != nil { return nil, errno } // Register pipe fd with epoll - event = syscall.EpollEvent{ + event = unix.EpollEvent{ Fd: int32(poller.pipe[0]), - Events: syscall.EPOLLIN, + Events: unix.EPOLLIN, } - errno = syscall.EpollCtl(poller.epfd, syscall.EPOLL_CTL_ADD, poller.pipe[0], &event) + errno = unix.EpollCtl(poller.epfd, unix.EPOLL_CTL_ADD, poller.pipe[0], &event) if errno != nil { return nil, errno } @@ -80,11 +81,11 @@ func (poller *fdPoller) wait() (bool, error) { // I don't know whether epoll_wait returns the number of events returned, // or the total number of events ready. // I decided to catch both by making the buffer one larger than the maximum. - events := make([]syscall.EpollEvent, 7) + events := make([]unix.EpollEvent, 7) for { - n, errno := syscall.EpollWait(poller.epfd, events, -1) + n, errno := unix.EpollWait(poller.epfd, events, -1) if n == -1 { - if errno == syscall.EINTR { + if errno == unix.EINTR { continue } return false, errno @@ -103,31 +104,31 @@ func (poller *fdPoller) wait() (bool, error) { epollin := false for _, event := range ready { if event.Fd == int32(poller.fd) { - if event.Events&syscall.EPOLLHUP != 0 { + if event.Events&unix.EPOLLHUP != 0 { // This should not happen, but if it does, treat it as a wakeup. epollhup = true } - if event.Events&syscall.EPOLLERR != 0 { + if event.Events&unix.EPOLLERR != 0 { // If an error is waiting on the file descriptor, we should pretend - // something is ready to read, and let syscall.Read pick up the error. + // something is ready to read, and let unix.Read pick up the error. epollerr = true } - if event.Events&syscall.EPOLLIN != 0 { + if event.Events&unix.EPOLLIN != 0 { // There is data to read. epollin = true } } if event.Fd == int32(poller.pipe[0]) { - if event.Events&syscall.EPOLLHUP != 0 { + if event.Events&unix.EPOLLHUP != 0 { // Write pipe descriptor was closed, by us. This means we're closing down the // watcher, and we should wake up. } - if event.Events&syscall.EPOLLERR != 0 { + if event.Events&unix.EPOLLERR != 0 { // If an error is waiting on the pipe file descriptor. // This is an absolute mystery, and should never ever happen. return false, errors.New("Error on the pipe descriptor.") } - if event.Events&syscall.EPOLLIN != 0 { + if event.Events&unix.EPOLLIN != 0 { // This is a regular wakeup, so we have to clear the buffer. err := poller.clearWake() if err != nil { @@ -147,9 +148,9 @@ func (poller *fdPoller) wait() (bool, error) { // Close the write end of the poller. func (poller *fdPoller) wake() error { buf := make([]byte, 1) - n, errno := syscall.Write(poller.pipe[1], buf) + n, errno := unix.Write(poller.pipe[1], buf) if n == -1 { - if errno == syscall.EAGAIN { + if errno == unix.EAGAIN { // Buffer is full, poller will wake. return nil } @@ -161,9 +162,9 @@ func (poller *fdPoller) wake() error { func (poller *fdPoller) clearWake() error { // You have to be woken up a LOT in order to get to 100! buf := make([]byte, 100) - n, errno := syscall.Read(poller.pipe[0], buf) + n, errno := unix.Read(poller.pipe[0], buf) if n == -1 { - if errno == syscall.EAGAIN { + if errno == unix.EAGAIN { // Buffer is empty, someone else cleared our wake. return nil } @@ -175,12 +176,12 @@ func (poller *fdPoller) clearWake() error { // Close all poller file descriptors, but not the one passed to it. func (poller *fdPoller) close() { if poller.pipe[1] != -1 { - syscall.Close(poller.pipe[1]) + unix.Close(poller.pipe[1]) } if poller.pipe[0] != -1 { - syscall.Close(poller.pipe[0]) + unix.Close(poller.pipe[0]) } if poller.epfd != -1 { - syscall.Close(poller.epfd) + unix.Close(poller.epfd) } } diff --git a/vendor/gopkg.in/fsnotify.v1/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go similarity index 90% rename from vendor/gopkg.in/fsnotify.v1/kqueue.go rename to vendor/github.com/fsnotify/fsnotify/kqueue.go index b8ea3084..c2b4acb1 100644 --- a/vendor/gopkg.in/fsnotify.v1/kqueue.go +++ b/vendor/github.com/fsnotify/fsnotify/kqueue.go @@ -13,8 +13,9 @@ import ( "os" "path/filepath" "sync" - "syscall" "time" + + "golang.org/x/sys/unix" ) // Watcher watches a set of files, delivering events to a channel. @@ -113,12 +114,12 @@ func (w *Watcher) Remove(name string) error { return fmt.Errorf("can't remove non-existent kevent watch for: %s", name) } - const registerRemove = syscall.EV_DELETE + const registerRemove = unix.EV_DELETE if err := register(w.kq, []int{watchfd}, registerRemove, 0); err != nil { return err } - syscall.Close(watchfd) + unix.Close(watchfd) w.mu.Lock() isDir := w.paths[watchfd].isDir @@ -152,7 +153,7 @@ func (w *Watcher) Remove(name string) error { } // Watch all events (except NOTE_EXTEND, NOTE_LINK, NOTE_REVOKE) -const noteAllEvents = syscall.NOTE_DELETE | syscall.NOTE_WRITE | syscall.NOTE_ATTRIB | syscall.NOTE_RENAME +const noteAllEvents = unix.NOTE_DELETE | unix.NOTE_WRITE | unix.NOTE_ATTRIB | unix.NOTE_RENAME // keventWaitTime to block on each read from kevent var keventWaitTime = durationToTimespec(100 * time.Millisecond) @@ -219,7 +220,7 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { } } - watchfd, err = syscall.Open(name, openMode, 0700) + watchfd, err = unix.Open(name, openMode, 0700) if watchfd == -1 { return "", err } @@ -227,9 +228,9 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { isDir = fi.IsDir() } - const registerAdd = syscall.EV_ADD | syscall.EV_CLEAR | syscall.EV_ENABLE + const registerAdd = unix.EV_ADD | unix.EV_CLEAR | unix.EV_ENABLE if err := register(w.kq, []int{watchfd}, registerAdd, flags); err != nil { - syscall.Close(watchfd) + unix.Close(watchfd) return "", err } @@ -245,8 +246,8 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { // or if it was watched before, but perhaps only a NOTE_DELETE (watchDirectoryFiles) w.mu.Lock() - watchDir := (flags&syscall.NOTE_WRITE) == syscall.NOTE_WRITE && - (!alreadyWatching || (w.dirFlags[name]&syscall.NOTE_WRITE) != syscall.NOTE_WRITE) + watchDir := (flags&unix.NOTE_WRITE) == unix.NOTE_WRITE && + (!alreadyWatching || (w.dirFlags[name]&unix.NOTE_WRITE) != unix.NOTE_WRITE) // Store flags so this watch can be updated later w.dirFlags[name] = flags w.mu.Unlock() @@ -263,13 +264,13 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { // readEvents reads from kqueue and converts the received kevents into // Event values that it sends down the Events channel. func (w *Watcher) readEvents() { - eventBuffer := make([]syscall.Kevent_t, 10) + eventBuffer := make([]unix.Kevent_t, 10) for { // See if there is a message on the "done" channel select { case <-w.done: - err := syscall.Close(w.kq) + err := unix.Close(w.kq) if err != nil { w.Errors <- err } @@ -282,7 +283,7 @@ func (w *Watcher) readEvents() { // Get new events kevents, err := read(w.kq, eventBuffer, &keventWaitTime) // EINTR is okay, the syscall was interrupted before timeout expired. - if err != nil && err != syscall.EINTR { + if err != nil && err != unix.EINTR { w.Errors <- err continue } @@ -356,16 +357,16 @@ func (w *Watcher) readEvents() { // newEvent returns an platform-independent Event based on kqueue Fflags. func newEvent(name string, mask uint32) Event { e := Event{Name: name} - if mask&syscall.NOTE_DELETE == syscall.NOTE_DELETE { + if mask&unix.NOTE_DELETE == unix.NOTE_DELETE { e.Op |= Remove } - if mask&syscall.NOTE_WRITE == syscall.NOTE_WRITE { + if mask&unix.NOTE_WRITE == unix.NOTE_WRITE { e.Op |= Write } - if mask&syscall.NOTE_RENAME == syscall.NOTE_RENAME { + if mask&unix.NOTE_RENAME == unix.NOTE_RENAME { e.Op |= Rename } - if mask&syscall.NOTE_ATTRIB == syscall.NOTE_ATTRIB { + if mask&unix.NOTE_ATTRIB == unix.NOTE_ATTRIB { e.Op |= Chmod } return e @@ -451,7 +452,7 @@ func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, erro flags := w.dirFlags[name] w.mu.Unlock() - flags |= syscall.NOTE_DELETE | syscall.NOTE_RENAME + flags |= unix.NOTE_DELETE | unix.NOTE_RENAME return w.addWatch(name, flags) } @@ -461,7 +462,7 @@ func (w *Watcher) internalWatch(name string, fileInfo os.FileInfo) (string, erro // kqueue creates a new kernel event queue and returns a descriptor. func kqueue() (kq int, err error) { - kq, err = syscall.Kqueue() + kq, err = unix.Kqueue() if kq == -1 { return kq, err } @@ -470,16 +471,16 @@ func kqueue() (kq int, err error) { // register events with the queue func register(kq int, fds []int, flags int, fflags uint32) error { - changes := make([]syscall.Kevent_t, len(fds)) + changes := make([]unix.Kevent_t, len(fds)) for i, fd := range fds { // SetKevent converts int to the platform-specific types: - syscall.SetKevent(&changes[i], fd, syscall.EVFILT_VNODE, flags) + unix.SetKevent(&changes[i], fd, unix.EVFILT_VNODE, flags) changes[i].Fflags = fflags } // register the events - success, err := syscall.Kevent(kq, changes, nil, nil) + success, err := unix.Kevent(kq, changes, nil, nil) if success == -1 { return err } @@ -488,8 +489,8 @@ func register(kq int, fds []int, flags int, fflags uint32) error { // read retrieves pending events, or waits until an event occurs. // A timeout of nil blocks indefinitely, while 0 polls the queue. -func read(kq int, events []syscall.Kevent_t, timeout *syscall.Timespec) ([]syscall.Kevent_t, error) { - n, err := syscall.Kevent(kq, nil, events, timeout) +func read(kq int, events []unix.Kevent_t, timeout *unix.Timespec) ([]unix.Kevent_t, error) { + n, err := unix.Kevent(kq, nil, events, timeout) if err != nil { return nil, err } @@ -497,6 +498,6 @@ func read(kq int, events []syscall.Kevent_t, timeout *syscall.Timespec) ([]sysca } // durationToTimespec prepares a timeout value -func durationToTimespec(d time.Duration) syscall.Timespec { - return syscall.NsecToTimespec(d.Nanoseconds()) +func durationToTimespec(d time.Duration) unix.Timespec { + return unix.NsecToTimespec(d.Nanoseconds()) } diff --git a/vendor/gopkg.in/fsnotify.v1/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go similarity index 73% rename from vendor/gopkg.in/fsnotify.v1/open_mode_bsd.go rename to vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go index c57ccb42..7d8de145 100644 --- a/vendor/gopkg.in/fsnotify.v1/open_mode_bsd.go +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go @@ -6,6 +6,6 @@ package fsnotify -import "syscall" +import "golang.org/x/sys/unix" -const openMode = syscall.O_NONBLOCK | syscall.O_RDONLY +const openMode = unix.O_NONBLOCK | unix.O_RDONLY diff --git a/vendor/gopkg.in/fsnotify.v1/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go similarity index 79% rename from vendor/gopkg.in/fsnotify.v1/open_mode_darwin.go rename to vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go index 174b2c33..9139e171 100644 --- a/vendor/gopkg.in/fsnotify.v1/open_mode_darwin.go +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go @@ -6,7 +6,7 @@ package fsnotify -import "syscall" +import "golang.org/x/sys/unix" // note: this constant is not defined on BSD -const openMode = syscall.O_EVTONLY +const openMode = unix.O_EVTONLY diff --git a/vendor/gopkg.in/fsnotify.v1/windows.go b/vendor/github.com/fsnotify/fsnotify/windows.go similarity index 86% rename from vendor/gopkg.in/fsnotify.v1/windows.go rename to vendor/github.com/fsnotify/fsnotify/windows.go index 81158522..09436f31 100644 --- a/vendor/gopkg.in/fsnotify.v1/windows.go +++ b/vendor/github.com/fsnotify/fsnotify/windows.go @@ -71,7 +71,7 @@ func (w *Watcher) Add(name string) error { in := &input{ op: opAddWatch, path: filepath.Clean(name), - flags: sys_FS_ALL_EVENTS, + flags: sysFSALLEVENTS, reply: make(chan error), } w.input <- in @@ -97,43 +97,43 @@ func (w *Watcher) Remove(name string) error { const ( // Options for AddWatch - sys_FS_ONESHOT = 0x80000000 - sys_FS_ONLYDIR = 0x1000000 + sysFSONESHOT = 0x80000000 + sysFSONLYDIR = 0x1000000 // Events - sys_FS_ACCESS = 0x1 - sys_FS_ALL_EVENTS = 0xfff - sys_FS_ATTRIB = 0x4 - sys_FS_CLOSE = 0x18 - sys_FS_CREATE = 0x100 - sys_FS_DELETE = 0x200 - sys_FS_DELETE_SELF = 0x400 - sys_FS_MODIFY = 0x2 - sys_FS_MOVE = 0xc0 - sys_FS_MOVED_FROM = 0x40 - sys_FS_MOVED_TO = 0x80 - sys_FS_MOVE_SELF = 0x800 + sysFSACCESS = 0x1 + sysFSALLEVENTS = 0xfff + sysFSATTRIB = 0x4 + sysFSCLOSE = 0x18 + sysFSCREATE = 0x100 + sysFSDELETE = 0x200 + sysFSDELETESELF = 0x400 + sysFSMODIFY = 0x2 + sysFSMOVE = 0xc0 + sysFSMOVEDFROM = 0x40 + sysFSMOVEDTO = 0x80 + sysFSMOVESELF = 0x800 // Special events - sys_FS_IGNORED = 0x8000 - sys_FS_Q_OVERFLOW = 0x4000 + sysFSIGNORED = 0x8000 + sysFSQOVERFLOW = 0x4000 ) func newEvent(name string, mask uint32) Event { e := Event{Name: name} - if mask&sys_FS_CREATE == sys_FS_CREATE || mask&sys_FS_MOVED_TO == sys_FS_MOVED_TO { + if mask&sysFSCREATE == sysFSCREATE || mask&sysFSMOVEDTO == sysFSMOVEDTO { e.Op |= Create } - if mask&sys_FS_DELETE == sys_FS_DELETE || mask&sys_FS_DELETE_SELF == sys_FS_DELETE_SELF { + if mask&sysFSDELETE == sysFSDELETE || mask&sysFSDELETESELF == sysFSDELETESELF { e.Op |= Remove } - if mask&sys_FS_MODIFY == sys_FS_MODIFY { + if mask&sysFSMODIFY == sysFSMODIFY { e.Op |= Write } - if mask&sys_FS_MOVE == sys_FS_MOVE || mask&sys_FS_MOVE_SELF == sys_FS_MOVE_SELF || mask&sys_FS_MOVED_FROM == sys_FS_MOVED_FROM { + if mask&sysFSMOVE == sysFSMOVE || mask&sysFSMOVESELF == sysFSMOVESELF || mask&sysFSMOVEDFROM == sysFSMOVEDFROM { e.Op |= Rename } - if mask&sys_FS_ATTRIB == sys_FS_ATTRIB { + if mask&sysFSATTRIB == sysFSATTRIB { e.Op |= Chmod } return e @@ -242,7 +242,7 @@ func (w *Watcher) addWatch(pathname string, flags uint64) error { if err != nil { return err } - if flags&sys_FS_ONLYDIR != 0 && pathname != dir { + if flags&sysFSONLYDIR != 0 && pathname != dir { return nil } ino, err := getIno(dir) @@ -302,11 +302,11 @@ func (w *Watcher) remWatch(pathname string) error { return fmt.Errorf("can't remove non-existent watch for: %s", pathname) } if pathname == dir { - w.sendEvent(watch.path, watch.mask&sys_FS_IGNORED) + w.sendEvent(watch.path, watch.mask&sysFSIGNORED) watch.mask = 0 } else { name := filepath.Base(pathname) - w.sendEvent(watch.path+"\\"+name, watch.names[name]&sys_FS_IGNORED) + w.sendEvent(filepath.Join(watch.path, name), watch.names[name]&sysFSIGNORED) delete(watch.names, name) } return w.startRead(watch) @@ -316,13 +316,13 @@ func (w *Watcher) remWatch(pathname string) error { func (w *Watcher) deleteWatch(watch *watch) { for name, mask := range watch.names { if mask&provisional == 0 { - w.sendEvent(watch.path+"\\"+name, mask&sys_FS_IGNORED) + w.sendEvent(filepath.Join(watch.path, name), mask&sysFSIGNORED) } delete(watch.names, name) } if watch.mask != 0 { if watch.mask&provisional == 0 { - w.sendEvent(watch.path, watch.mask&sys_FS_IGNORED) + w.sendEvent(watch.path, watch.mask&sysFSIGNORED) } watch.mask = 0 } @@ -353,8 +353,8 @@ func (w *Watcher) startRead(watch *watch) error { err := os.NewSyscallError("ReadDirectoryChanges", e) if e == syscall.ERROR_ACCESS_DENIED && watch.mask&provisional == 0 { // Watched directory was probably removed - if w.sendEvent(watch.path, watch.mask&sys_FS_DELETE_SELF) { - if watch.mask&sys_FS_ONESHOT != 0 { + if w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) { + if watch.mask&sysFSONESHOT != 0 { watch.mask = 0 } } @@ -428,7 +428,7 @@ func (w *Watcher) readEvents() { } case syscall.ERROR_ACCESS_DENIED: // Watched directory was probably removed - w.sendEvent(watch.path, watch.mask&sys_FS_DELETE_SELF) + w.sendEvent(watch.path, watch.mask&sysFSDELETESELF) w.deleteWatch(watch) w.startRead(watch) continue @@ -444,7 +444,7 @@ func (w *Watcher) readEvents() { var offset uint32 for { if n == 0 { - w.Events <- newEvent("", sys_FS_Q_OVERFLOW) + w.Events <- newEvent("", sysFSQOVERFLOW) w.Errors <- errors.New("short read in readEvents()") break } @@ -453,27 +453,27 @@ func (w *Watcher) readEvents() { raw := (*syscall.FileNotifyInformation)(unsafe.Pointer(&watch.buf[offset])) buf := (*[syscall.MAX_PATH]uint16)(unsafe.Pointer(&raw.FileName)) name := syscall.UTF16ToString(buf[:raw.FileNameLength/2]) - fullname := watch.path + "\\" + name + fullname := filepath.Join(watch.path, name) var mask uint64 switch raw.Action { case syscall.FILE_ACTION_REMOVED: - mask = sys_FS_DELETE_SELF + mask = sysFSDELETESELF case syscall.FILE_ACTION_MODIFIED: - mask = sys_FS_MODIFY + mask = sysFSMODIFY case syscall.FILE_ACTION_RENAMED_OLD_NAME: watch.rename = name case syscall.FILE_ACTION_RENAMED_NEW_NAME: if watch.names[watch.rename] != 0 { watch.names[name] |= watch.names[watch.rename] delete(watch.names, watch.rename) - mask = sys_FS_MOVE_SELF + mask = sysFSMOVESELF } } sendNameEvent := func() { if w.sendEvent(fullname, watch.names[name]&mask) { - if watch.names[name]&sys_FS_ONESHOT != 0 { + if watch.names[name]&sysFSONESHOT != 0 { delete(watch.names, name) } } @@ -482,16 +482,16 @@ func (w *Watcher) readEvents() { sendNameEvent() } if raw.Action == syscall.FILE_ACTION_REMOVED { - w.sendEvent(fullname, watch.names[name]&sys_FS_IGNORED) + w.sendEvent(fullname, watch.names[name]&sysFSIGNORED) delete(watch.names, name) } if w.sendEvent(fullname, watch.mask&toFSnotifyFlags(raw.Action)) { - if watch.mask&sys_FS_ONESHOT != 0 { + if watch.mask&sysFSONESHOT != 0 { watch.mask = 0 } } if raw.Action == syscall.FILE_ACTION_RENAMED_NEW_NAME { - fullname = watch.path + "\\" + watch.rename + fullname = filepath.Join(watch.path, watch.rename) sendNameEvent() } @@ -529,16 +529,16 @@ func (w *Watcher) sendEvent(name string, mask uint64) bool { func toWindowsFlags(mask uint64) uint32 { var m uint32 - if mask&sys_FS_ACCESS != 0 { + if mask&sysFSACCESS != 0 { m |= syscall.FILE_NOTIFY_CHANGE_LAST_ACCESS } - if mask&sys_FS_MODIFY != 0 { + if mask&sysFSMODIFY != 0 { m |= syscall.FILE_NOTIFY_CHANGE_LAST_WRITE } - if mask&sys_FS_ATTRIB != 0 { + if mask&sysFSATTRIB != 0 { m |= syscall.FILE_NOTIFY_CHANGE_ATTRIBUTES } - if mask&(sys_FS_MOVE|sys_FS_CREATE|sys_FS_DELETE) != 0 { + if mask&(sysFSMOVE|sysFSCREATE|sysFSDELETE) != 0 { m |= syscall.FILE_NOTIFY_CHANGE_FILE_NAME | syscall.FILE_NOTIFY_CHANGE_DIR_NAME } return m @@ -547,15 +547,15 @@ func toWindowsFlags(mask uint64) uint32 { func toFSnotifyFlags(action uint32) uint64 { switch action { case syscall.FILE_ACTION_ADDED: - return sys_FS_CREATE + return sysFSCREATE case syscall.FILE_ACTION_REMOVED: - return sys_FS_DELETE + return sysFSDELETE case syscall.FILE_ACTION_MODIFIED: - return sys_FS_MODIFY + return sysFSMODIFY case syscall.FILE_ACTION_RENAMED_OLD_NAME: - return sys_FS_MOVED_FROM + return sysFSMOVEDFROM case syscall.FILE_ACTION_RENAMED_NEW_NAME: - return sys_FS_MOVED_TO + return sysFSMOVEDTO } return 0 } diff --git a/vendor/github.com/golang/protobuf/LICENSE b/vendor/github.com/golang/protobuf/LICENSE new file mode 100644 index 00000000..1b1b1921 --- /dev/null +++ b/vendor/github.com/golang/protobuf/LICENSE @@ -0,0 +1,31 @@ +Go support for Protocol Buffers - Google's data interchange format + +Copyright 2010 The Go Authors. All rights reserved. +https://github.com/golang/protobuf + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/golang/protobuf/proto/Makefile b/vendor/github.com/golang/protobuf/proto/Makefile index f1f06564..e2e0651a 100644 --- a/vendor/github.com/golang/protobuf/proto/Makefile +++ b/vendor/github.com/golang/protobuf/proto/Makefile @@ -39,5 +39,5 @@ test: install generate-test-pbs generate-test-pbs: make install make -C testdata - protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata:. proto3_proto/proto3.proto + protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto make diff --git a/vendor/github.com/golang/protobuf/proto/all_test.go b/vendor/github.com/golang/protobuf/proto/all_test.go deleted file mode 100644 index f68f913d..00000000 --- a/vendor/github.com/golang/protobuf/proto/all_test.go +++ /dev/null @@ -1,2169 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2010 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "math" - "math/rand" - "reflect" - "runtime/debug" - "strings" - "testing" - "time" - - . "github.com/golang/protobuf/proto" - . "github.com/golang/protobuf/proto/testdata" -) - -var globalO *Buffer - -func old() *Buffer { - if globalO == nil { - globalO = NewBuffer(nil) - } - globalO.Reset() - return globalO -} - -func equalbytes(b1, b2 []byte, t *testing.T) { - if len(b1) != len(b2) { - t.Errorf("wrong lengths: 2*%d != %d", len(b1), len(b2)) - return - } - for i := 0; i < len(b1); i++ { - if b1[i] != b2[i] { - t.Errorf("bad byte[%d]:%x %x: %s %s", i, b1[i], b2[i], b1, b2) - } - } -} - -func initGoTestField() *GoTestField { - f := new(GoTestField) - f.Label = String("label") - f.Type = String("type") - return f -} - -// These are all structurally equivalent but the tag numbers differ. -// (It's remarkable that required, optional, and repeated all have -// 8 letters.) -func initGoTest_RequiredGroup() *GoTest_RequiredGroup { - return &GoTest_RequiredGroup{ - RequiredField: String("required"), - } -} - -func initGoTest_OptionalGroup() *GoTest_OptionalGroup { - return &GoTest_OptionalGroup{ - RequiredField: String("optional"), - } -} - -func initGoTest_RepeatedGroup() *GoTest_RepeatedGroup { - return &GoTest_RepeatedGroup{ - RequiredField: String("repeated"), - } -} - -func initGoTest(setdefaults bool) *GoTest { - pb := new(GoTest) - if setdefaults { - pb.F_BoolDefaulted = Bool(Default_GoTest_F_BoolDefaulted) - pb.F_Int32Defaulted = Int32(Default_GoTest_F_Int32Defaulted) - pb.F_Int64Defaulted = Int64(Default_GoTest_F_Int64Defaulted) - pb.F_Fixed32Defaulted = Uint32(Default_GoTest_F_Fixed32Defaulted) - pb.F_Fixed64Defaulted = Uint64(Default_GoTest_F_Fixed64Defaulted) - pb.F_Uint32Defaulted = Uint32(Default_GoTest_F_Uint32Defaulted) - pb.F_Uint64Defaulted = Uint64(Default_GoTest_F_Uint64Defaulted) - pb.F_FloatDefaulted = Float32(Default_GoTest_F_FloatDefaulted) - pb.F_DoubleDefaulted = Float64(Default_GoTest_F_DoubleDefaulted) - pb.F_StringDefaulted = String(Default_GoTest_F_StringDefaulted) - pb.F_BytesDefaulted = Default_GoTest_F_BytesDefaulted - pb.F_Sint32Defaulted = Int32(Default_GoTest_F_Sint32Defaulted) - pb.F_Sint64Defaulted = Int64(Default_GoTest_F_Sint64Defaulted) - } - - pb.Kind = GoTest_TIME.Enum() - pb.RequiredField = initGoTestField() - pb.F_BoolRequired = Bool(true) - pb.F_Int32Required = Int32(3) - pb.F_Int64Required = Int64(6) - pb.F_Fixed32Required = Uint32(32) - pb.F_Fixed64Required = Uint64(64) - pb.F_Uint32Required = Uint32(3232) - pb.F_Uint64Required = Uint64(6464) - pb.F_FloatRequired = Float32(3232) - pb.F_DoubleRequired = Float64(6464) - pb.F_StringRequired = String("string") - pb.F_BytesRequired = []byte("bytes") - pb.F_Sint32Required = Int32(-32) - pb.F_Sint64Required = Int64(-64) - pb.Requiredgroup = initGoTest_RequiredGroup() - - return pb -} - -func fail(msg string, b *bytes.Buffer, s string, t *testing.T) { - data := b.Bytes() - ld := len(data) - ls := len(s) / 2 - - fmt.Printf("fail %s ld=%d ls=%d\n", msg, ld, ls) - - // find the interesting spot - n - n := ls - if ld < ls { - n = ld - } - j := 0 - for i := 0; i < n; i++ { - bs := hex(s[j])*16 + hex(s[j+1]) - j += 2 - if data[i] == bs { - continue - } - n = i - break - } - l := n - 10 - if l < 0 { - l = 0 - } - h := n + 10 - - // find the interesting spot - n - fmt.Printf("is[%d]:", l) - for i := l; i < h; i++ { - if i >= ld { - fmt.Printf(" --") - continue - } - fmt.Printf(" %.2x", data[i]) - } - fmt.Printf("\n") - - fmt.Printf("sb[%d]:", l) - for i := l; i < h; i++ { - if i >= ls { - fmt.Printf(" --") - continue - } - bs := hex(s[j])*16 + hex(s[j+1]) - j += 2 - fmt.Printf(" %.2x", bs) - } - fmt.Printf("\n") - - t.Fail() - - // t.Errorf("%s: \ngood: %s\nbad: %x", msg, s, b.Bytes()) - // Print the output in a partially-decoded format; can - // be helpful when updating the test. It produces the output - // that is pasted, with minor edits, into the argument to verify(). - // data := b.Bytes() - // nesting := 0 - // for b.Len() > 0 { - // start := len(data) - b.Len() - // var u uint64 - // u, err := DecodeVarint(b) - // if err != nil { - // fmt.Printf("decode error on varint:", err) - // return - // } - // wire := u & 0x7 - // tag := u >> 3 - // switch wire { - // case WireVarint: - // v, err := DecodeVarint(b) - // if err != nil { - // fmt.Printf("decode error on varint:", err) - // return - // } - // fmt.Printf("\t\t\"%x\" // field %d, encoding %d, value %d\n", - // data[start:len(data)-b.Len()], tag, wire, v) - // case WireFixed32: - // v, err := DecodeFixed32(b) - // if err != nil { - // fmt.Printf("decode error on fixed32:", err) - // return - // } - // fmt.Printf("\t\t\"%x\" // field %d, encoding %d, value %d\n", - // data[start:len(data)-b.Len()], tag, wire, v) - // case WireFixed64: - // v, err := DecodeFixed64(b) - // if err != nil { - // fmt.Printf("decode error on fixed64:", err) - // return - // } - // fmt.Printf("\t\t\"%x\" // field %d, encoding %d, value %d\n", - // data[start:len(data)-b.Len()], tag, wire, v) - // case WireBytes: - // nb, err := DecodeVarint(b) - // if err != nil { - // fmt.Printf("decode error on bytes:", err) - // return - // } - // after_tag := len(data) - b.Len() - // str := make([]byte, nb) - // _, err = b.Read(str) - // if err != nil { - // fmt.Printf("decode error on bytes:", err) - // return - // } - // fmt.Printf("\t\t\"%x\" \"%x\" // field %d, encoding %d (FIELD)\n", - // data[start:after_tag], str, tag, wire) - // case WireStartGroup: - // nesting++ - // fmt.Printf("\t\t\"%x\"\t\t// start group field %d level %d\n", - // data[start:len(data)-b.Len()], tag, nesting) - // case WireEndGroup: - // fmt.Printf("\t\t\"%x\"\t\t// end group field %d level %d\n", - // data[start:len(data)-b.Len()], tag, nesting) - // nesting-- - // default: - // fmt.Printf("unrecognized wire type %d\n", wire) - // return - // } - // } -} - -func hex(c uint8) uint8 { - if '0' <= c && c <= '9' { - return c - '0' - } - if 'a' <= c && c <= 'f' { - return 10 + c - 'a' - } - if 'A' <= c && c <= 'F' { - return 10 + c - 'A' - } - return 0 -} - -func equal(b []byte, s string, t *testing.T) bool { - if 2*len(b) != len(s) { - // fail(fmt.Sprintf("wrong lengths: 2*%d != %d", len(b), len(s)), b, s, t) - fmt.Printf("wrong lengths: 2*%d != %d\n", len(b), len(s)) - return false - } - for i, j := 0, 0; i < len(b); i, j = i+1, j+2 { - x := hex(s[j])*16 + hex(s[j+1]) - if b[i] != x { - // fail(fmt.Sprintf("bad byte[%d]:%x %x", i, b[i], x), b, s, t) - fmt.Printf("bad byte[%d]:%x %x", i, b[i], x) - return false - } - } - return true -} - -func overify(t *testing.T, pb *GoTest, expected string) { - o := old() - err := o.Marshal(pb) - if err != nil { - fmt.Printf("overify marshal-1 err = %v", err) - o.DebugPrint("", o.Bytes()) - t.Fatalf("expected = %s", expected) - } - if !equal(o.Bytes(), expected, t) { - o.DebugPrint("overify neq 1", o.Bytes()) - t.Fatalf("expected = %s", expected) - } - - // Now test Unmarshal by recreating the original buffer. - pbd := new(GoTest) - err = o.Unmarshal(pbd) - if err != nil { - t.Fatalf("overify unmarshal err = %v", err) - o.DebugPrint("", o.Bytes()) - t.Fatalf("string = %s", expected) - } - o.Reset() - err = o.Marshal(pbd) - if err != nil { - t.Errorf("overify marshal-2 err = %v", err) - o.DebugPrint("", o.Bytes()) - t.Fatalf("string = %s", expected) - } - if !equal(o.Bytes(), expected, t) { - o.DebugPrint("overify neq 2", o.Bytes()) - t.Fatalf("string = %s", expected) - } -} - -// Simple tests for numeric encode/decode primitives (varint, etc.) -func TestNumericPrimitives(t *testing.T) { - for i := uint64(0); i < 1e6; i += 111 { - o := old() - if o.EncodeVarint(i) != nil { - t.Error("EncodeVarint") - break - } - x, e := o.DecodeVarint() - if e != nil { - t.Fatal("DecodeVarint") - } - if x != i { - t.Fatal("varint decode fail:", i, x) - } - - o = old() - if o.EncodeFixed32(i) != nil { - t.Fatal("encFixed32") - } - x, e = o.DecodeFixed32() - if e != nil { - t.Fatal("decFixed32") - } - if x != i { - t.Fatal("fixed32 decode fail:", i, x) - } - - o = old() - if o.EncodeFixed64(i*1234567) != nil { - t.Error("encFixed64") - break - } - x, e = o.DecodeFixed64() - if e != nil { - t.Error("decFixed64") - break - } - if x != i*1234567 { - t.Error("fixed64 decode fail:", i*1234567, x) - break - } - - o = old() - i32 := int32(i - 12345) - if o.EncodeZigzag32(uint64(i32)) != nil { - t.Fatal("EncodeZigzag32") - } - x, e = o.DecodeZigzag32() - if e != nil { - t.Fatal("DecodeZigzag32") - } - if x != uint64(uint32(i32)) { - t.Fatal("zigzag32 decode fail:", i32, x) - } - - o = old() - i64 := int64(i - 12345) - if o.EncodeZigzag64(uint64(i64)) != nil { - t.Fatal("EncodeZigzag64") - } - x, e = o.DecodeZigzag64() - if e != nil { - t.Fatal("DecodeZigzag64") - } - if x != uint64(i64) { - t.Fatal("zigzag64 decode fail:", i64, x) - } - } -} - -// fakeMarshaler is a simple struct implementing Marshaler and Message interfaces. -type fakeMarshaler struct { - b []byte - err error -} - -func (f *fakeMarshaler) Marshal() ([]byte, error) { return f.b, f.err } -func (f *fakeMarshaler) String() string { return fmt.Sprintf("Bytes: %v Error: %v", f.b, f.err) } -func (f *fakeMarshaler) ProtoMessage() {} -func (f *fakeMarshaler) Reset() {} - -type msgWithFakeMarshaler struct { - M *fakeMarshaler `protobuf:"bytes,1,opt,name=fake"` -} - -func (m *msgWithFakeMarshaler) String() string { return CompactTextString(m) } -func (m *msgWithFakeMarshaler) ProtoMessage() {} -func (m *msgWithFakeMarshaler) Reset() {} - -// Simple tests for proto messages that implement the Marshaler interface. -func TestMarshalerEncoding(t *testing.T) { - tests := []struct { - name string - m Message - want []byte - wantErr error - }{ - { - name: "Marshaler that fails", - m: &fakeMarshaler{ - err: errors.New("some marshal err"), - b: []byte{5, 6, 7}, - }, - // Since there's an error, nothing should be written to buffer. - want: nil, - wantErr: errors.New("some marshal err"), - }, - { - name: "Marshaler that fails with RequiredNotSetError", - m: &msgWithFakeMarshaler{ - M: &fakeMarshaler{ - err: &RequiredNotSetError{}, - b: []byte{5, 6, 7}, - }, - }, - // Since there's an error that can be continued after, - // the buffer should be written. - want: []byte{ - 10, 3, // for &msgWithFakeMarshaler - 5, 6, 7, // for &fakeMarshaler - }, - wantErr: &RequiredNotSetError{}, - }, - { - name: "Marshaler that succeeds", - m: &fakeMarshaler{ - b: []byte{0, 1, 2, 3, 4, 127, 255}, - }, - want: []byte{0, 1, 2, 3, 4, 127, 255}, - wantErr: nil, - }, - } - for _, test := range tests { - b := NewBuffer(nil) - err := b.Marshal(test.m) - if _, ok := err.(*RequiredNotSetError); ok { - // We're not in package proto, so we can only assert the type in this case. - err = &RequiredNotSetError{} - } - if !reflect.DeepEqual(test.wantErr, err) { - t.Errorf("%s: got err %v wanted %v", test.name, err, test.wantErr) - } - if !reflect.DeepEqual(test.want, b.Bytes()) { - t.Errorf("%s: got bytes %v wanted %v", test.name, b.Bytes(), test.want) - } - } -} - -// Simple tests for bytes -func TestBytesPrimitives(t *testing.T) { - o := old() - bytes := []byte{'n', 'o', 'w', ' ', 'i', 's', ' ', 't', 'h', 'e', ' ', 't', 'i', 'm', 'e'} - if o.EncodeRawBytes(bytes) != nil { - t.Error("EncodeRawBytes") - } - decb, e := o.DecodeRawBytes(false) - if e != nil { - t.Error("DecodeRawBytes") - } - equalbytes(bytes, decb, t) -} - -// Simple tests for strings -func TestStringPrimitives(t *testing.T) { - o := old() - s := "now is the time" - if o.EncodeStringBytes(s) != nil { - t.Error("enc_string") - } - decs, e := o.DecodeStringBytes() - if e != nil { - t.Error("dec_string") - } - if s != decs { - t.Error("string encode/decode fail:", s, decs) - } -} - -// Do we catch the "required bit not set" case? -func TestRequiredBit(t *testing.T) { - o := old() - pb := new(GoTest) - err := o.Marshal(pb) - if err == nil { - t.Error("did not catch missing required fields") - } else if strings.Index(err.Error(), "Kind") < 0 { - t.Error("wrong error type:", err) - } -} - -// Check that all fields are nil. -// Clearly silly, and a residue from a more interesting test with an earlier, -// different initialization property, but it once caught a compiler bug so -// it lives. -func checkInitialized(pb *GoTest, t *testing.T) { - if pb.F_BoolDefaulted != nil { - t.Error("New or Reset did not set boolean:", *pb.F_BoolDefaulted) - } - if pb.F_Int32Defaulted != nil { - t.Error("New or Reset did not set int32:", *pb.F_Int32Defaulted) - } - if pb.F_Int64Defaulted != nil { - t.Error("New or Reset did not set int64:", *pb.F_Int64Defaulted) - } - if pb.F_Fixed32Defaulted != nil { - t.Error("New or Reset did not set fixed32:", *pb.F_Fixed32Defaulted) - } - if pb.F_Fixed64Defaulted != nil { - t.Error("New or Reset did not set fixed64:", *pb.F_Fixed64Defaulted) - } - if pb.F_Uint32Defaulted != nil { - t.Error("New or Reset did not set uint32:", *pb.F_Uint32Defaulted) - } - if pb.F_Uint64Defaulted != nil { - t.Error("New or Reset did not set uint64:", *pb.F_Uint64Defaulted) - } - if pb.F_FloatDefaulted != nil { - t.Error("New or Reset did not set float:", *pb.F_FloatDefaulted) - } - if pb.F_DoubleDefaulted != nil { - t.Error("New or Reset did not set double:", *pb.F_DoubleDefaulted) - } - if pb.F_StringDefaulted != nil { - t.Error("New or Reset did not set string:", *pb.F_StringDefaulted) - } - if pb.F_BytesDefaulted != nil { - t.Error("New or Reset did not set bytes:", string(pb.F_BytesDefaulted)) - } - if pb.F_Sint32Defaulted != nil { - t.Error("New or Reset did not set int32:", *pb.F_Sint32Defaulted) - } - if pb.F_Sint64Defaulted != nil { - t.Error("New or Reset did not set int64:", *pb.F_Sint64Defaulted) - } -} - -// Does Reset() reset? -func TestReset(t *testing.T) { - pb := initGoTest(true) - // muck with some values - pb.F_BoolDefaulted = Bool(false) - pb.F_Int32Defaulted = Int32(237) - pb.F_Int64Defaulted = Int64(12346) - pb.F_Fixed32Defaulted = Uint32(32000) - pb.F_Fixed64Defaulted = Uint64(666) - pb.F_Uint32Defaulted = Uint32(323232) - pb.F_Uint64Defaulted = nil - pb.F_FloatDefaulted = nil - pb.F_DoubleDefaulted = Float64(0) - pb.F_StringDefaulted = String("gotcha") - pb.F_BytesDefaulted = []byte("asdfasdf") - pb.F_Sint32Defaulted = Int32(123) - pb.F_Sint64Defaulted = Int64(789) - pb.Reset() - checkInitialized(pb, t) -} - -// All required fields set, no defaults provided. -func TestEncodeDecode1(t *testing.T) { - pb := initGoTest(false) - overify(t, pb, - "0807"+ // field 1, encoding 0, value 7 - "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) - "5001"+ // field 10, encoding 0, value 1 - "5803"+ // field 11, encoding 0, value 3 - "6006"+ // field 12, encoding 0, value 6 - "6d20000000"+ // field 13, encoding 5, value 0x20 - "714000000000000000"+ // field 14, encoding 1, value 0x40 - "78a019"+ // field 15, encoding 0, value 0xca0 = 3232 - "8001c032"+ // field 16, encoding 0, value 0x1940 = 6464 - "8d0100004a45"+ // field 17, encoding 5, value 3232.0 - "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 - "9a0106"+"737472696e67"+ // field 19, encoding 2, string "string" - "b304"+ // field 70, encoding 3, start group - "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" - "b404"+ // field 70, encoding 4, end group - "aa0605"+"6279746573"+ // field 101, encoding 2, string "bytes" - "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 - "b8067f") // field 103, encoding 0, 0x7f zigzag64 -} - -// All required fields set, defaults provided. -func TestEncodeDecode2(t *testing.T) { - pb := initGoTest(true) - overify(t, pb, - "0807"+ // field 1, encoding 0, value 7 - "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) - "5001"+ // field 10, encoding 0, value 1 - "5803"+ // field 11, encoding 0, value 3 - "6006"+ // field 12, encoding 0, value 6 - "6d20000000"+ // field 13, encoding 5, value 32 - "714000000000000000"+ // field 14, encoding 1, value 64 - "78a019"+ // field 15, encoding 0, value 3232 - "8001c032"+ // field 16, encoding 0, value 6464 - "8d0100004a45"+ // field 17, encoding 5, value 3232.0 - "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 - "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" - "c00201"+ // field 40, encoding 0, value 1 - "c80220"+ // field 41, encoding 0, value 32 - "d00240"+ // field 42, encoding 0, value 64 - "dd0240010000"+ // field 43, encoding 5, value 320 - "e1028002000000000000"+ // field 44, encoding 1, value 640 - "e8028019"+ // field 45, encoding 0, value 3200 - "f0028032"+ // field 46, encoding 0, value 6400 - "fd02e0659948"+ // field 47, encoding 5, value 314159.0 - "81030000000050971041"+ // field 48, encoding 1, value 271828.0 - "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" - "b304"+ // start group field 70 level 1 - "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" - "b404"+ // end group field 70 level 1 - "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" - "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 - "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 - "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" - "90193f"+ // field 402, encoding 0, value 63 - "98197f") // field 403, encoding 0, value 127 - -} - -// All default fields set to their default value by hand -func TestEncodeDecode3(t *testing.T) { - pb := initGoTest(false) - pb.F_BoolDefaulted = Bool(true) - pb.F_Int32Defaulted = Int32(32) - pb.F_Int64Defaulted = Int64(64) - pb.F_Fixed32Defaulted = Uint32(320) - pb.F_Fixed64Defaulted = Uint64(640) - pb.F_Uint32Defaulted = Uint32(3200) - pb.F_Uint64Defaulted = Uint64(6400) - pb.F_FloatDefaulted = Float32(314159) - pb.F_DoubleDefaulted = Float64(271828) - pb.F_StringDefaulted = String("hello, \"world!\"\n") - pb.F_BytesDefaulted = []byte("Bignose") - pb.F_Sint32Defaulted = Int32(-32) - pb.F_Sint64Defaulted = Int64(-64) - - overify(t, pb, - "0807"+ // field 1, encoding 0, value 7 - "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) - "5001"+ // field 10, encoding 0, value 1 - "5803"+ // field 11, encoding 0, value 3 - "6006"+ // field 12, encoding 0, value 6 - "6d20000000"+ // field 13, encoding 5, value 32 - "714000000000000000"+ // field 14, encoding 1, value 64 - "78a019"+ // field 15, encoding 0, value 3232 - "8001c032"+ // field 16, encoding 0, value 6464 - "8d0100004a45"+ // field 17, encoding 5, value 3232.0 - "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 - "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" - "c00201"+ // field 40, encoding 0, value 1 - "c80220"+ // field 41, encoding 0, value 32 - "d00240"+ // field 42, encoding 0, value 64 - "dd0240010000"+ // field 43, encoding 5, value 320 - "e1028002000000000000"+ // field 44, encoding 1, value 640 - "e8028019"+ // field 45, encoding 0, value 3200 - "f0028032"+ // field 46, encoding 0, value 6400 - "fd02e0659948"+ // field 47, encoding 5, value 314159.0 - "81030000000050971041"+ // field 48, encoding 1, value 271828.0 - "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" - "b304"+ // start group field 70 level 1 - "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" - "b404"+ // end group field 70 level 1 - "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" - "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 - "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 - "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" - "90193f"+ // field 402, encoding 0, value 63 - "98197f") // field 403, encoding 0, value 127 - -} - -// All required fields set, defaults provided, all non-defaulted optional fields have values. -func TestEncodeDecode4(t *testing.T) { - pb := initGoTest(true) - pb.Table = String("hello") - pb.Param = Int32(7) - pb.OptionalField = initGoTestField() - pb.F_BoolOptional = Bool(true) - pb.F_Int32Optional = Int32(32) - pb.F_Int64Optional = Int64(64) - pb.F_Fixed32Optional = Uint32(3232) - pb.F_Fixed64Optional = Uint64(6464) - pb.F_Uint32Optional = Uint32(323232) - pb.F_Uint64Optional = Uint64(646464) - pb.F_FloatOptional = Float32(32.) - pb.F_DoubleOptional = Float64(64.) - pb.F_StringOptional = String("hello") - pb.F_BytesOptional = []byte("Bignose") - pb.F_Sint32Optional = Int32(-32) - pb.F_Sint64Optional = Int64(-64) - pb.Optionalgroup = initGoTest_OptionalGroup() - - overify(t, pb, - "0807"+ // field 1, encoding 0, value 7 - "1205"+"68656c6c6f"+ // field 2, encoding 2, string "hello" - "1807"+ // field 3, encoding 0, value 7 - "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) - "320d"+"0a056c6162656c120474797065"+ // field 6, encoding 2 (GoTestField) - "5001"+ // field 10, encoding 0, value 1 - "5803"+ // field 11, encoding 0, value 3 - "6006"+ // field 12, encoding 0, value 6 - "6d20000000"+ // field 13, encoding 5, value 32 - "714000000000000000"+ // field 14, encoding 1, value 64 - "78a019"+ // field 15, encoding 0, value 3232 - "8001c032"+ // field 16, encoding 0, value 6464 - "8d0100004a45"+ // field 17, encoding 5, value 3232.0 - "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 - "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" - "f00101"+ // field 30, encoding 0, value 1 - "f80120"+ // field 31, encoding 0, value 32 - "800240"+ // field 32, encoding 0, value 64 - "8d02a00c0000"+ // field 33, encoding 5, value 3232 - "91024019000000000000"+ // field 34, encoding 1, value 6464 - "9802a0dd13"+ // field 35, encoding 0, value 323232 - "a002c0ba27"+ // field 36, encoding 0, value 646464 - "ad0200000042"+ // field 37, encoding 5, value 32.0 - "b1020000000000005040"+ // field 38, encoding 1, value 64.0 - "ba0205"+"68656c6c6f"+ // field 39, encoding 2, string "hello" - "c00201"+ // field 40, encoding 0, value 1 - "c80220"+ // field 41, encoding 0, value 32 - "d00240"+ // field 42, encoding 0, value 64 - "dd0240010000"+ // field 43, encoding 5, value 320 - "e1028002000000000000"+ // field 44, encoding 1, value 640 - "e8028019"+ // field 45, encoding 0, value 3200 - "f0028032"+ // field 46, encoding 0, value 6400 - "fd02e0659948"+ // field 47, encoding 5, value 314159.0 - "81030000000050971041"+ // field 48, encoding 1, value 271828.0 - "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" - "b304"+ // start group field 70 level 1 - "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" - "b404"+ // end group field 70 level 1 - "d305"+ // start group field 90 level 1 - "da0508"+"6f7074696f6e616c"+ // field 91, encoding 2, string "optional" - "d405"+ // end group field 90 level 1 - "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" - "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 - "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 - "ea1207"+"4269676e6f7365"+ // field 301, encoding 2, string "Bignose" - "f0123f"+ // field 302, encoding 0, value 63 - "f8127f"+ // field 303, encoding 0, value 127 - "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" - "90193f"+ // field 402, encoding 0, value 63 - "98197f") // field 403, encoding 0, value 127 - -} - -// All required fields set, defaults provided, all repeated fields given two values. -func TestEncodeDecode5(t *testing.T) { - pb := initGoTest(true) - pb.RepeatedField = []*GoTestField{initGoTestField(), initGoTestField()} - pb.F_BoolRepeated = []bool{false, true} - pb.F_Int32Repeated = []int32{32, 33} - pb.F_Int64Repeated = []int64{64, 65} - pb.F_Fixed32Repeated = []uint32{3232, 3333} - pb.F_Fixed64Repeated = []uint64{6464, 6565} - pb.F_Uint32Repeated = []uint32{323232, 333333} - pb.F_Uint64Repeated = []uint64{646464, 656565} - pb.F_FloatRepeated = []float32{32., 33.} - pb.F_DoubleRepeated = []float64{64., 65.} - pb.F_StringRepeated = []string{"hello", "sailor"} - pb.F_BytesRepeated = [][]byte{[]byte("big"), []byte("nose")} - pb.F_Sint32Repeated = []int32{32, -32} - pb.F_Sint64Repeated = []int64{64, -64} - pb.Repeatedgroup = []*GoTest_RepeatedGroup{initGoTest_RepeatedGroup(), initGoTest_RepeatedGroup()} - - overify(t, pb, - "0807"+ // field 1, encoding 0, value 7 - "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) - "2a0d"+"0a056c6162656c120474797065"+ // field 5, encoding 2 (GoTestField) - "2a0d"+"0a056c6162656c120474797065"+ // field 5, encoding 2 (GoTestField) - "5001"+ // field 10, encoding 0, value 1 - "5803"+ // field 11, encoding 0, value 3 - "6006"+ // field 12, encoding 0, value 6 - "6d20000000"+ // field 13, encoding 5, value 32 - "714000000000000000"+ // field 14, encoding 1, value 64 - "78a019"+ // field 15, encoding 0, value 3232 - "8001c032"+ // field 16, encoding 0, value 6464 - "8d0100004a45"+ // field 17, encoding 5, value 3232.0 - "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 - "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" - "a00100"+ // field 20, encoding 0, value 0 - "a00101"+ // field 20, encoding 0, value 1 - "a80120"+ // field 21, encoding 0, value 32 - "a80121"+ // field 21, encoding 0, value 33 - "b00140"+ // field 22, encoding 0, value 64 - "b00141"+ // field 22, encoding 0, value 65 - "bd01a00c0000"+ // field 23, encoding 5, value 3232 - "bd01050d0000"+ // field 23, encoding 5, value 3333 - "c1014019000000000000"+ // field 24, encoding 1, value 6464 - "c101a519000000000000"+ // field 24, encoding 1, value 6565 - "c801a0dd13"+ // field 25, encoding 0, value 323232 - "c80195ac14"+ // field 25, encoding 0, value 333333 - "d001c0ba27"+ // field 26, encoding 0, value 646464 - "d001b58928"+ // field 26, encoding 0, value 656565 - "dd0100000042"+ // field 27, encoding 5, value 32.0 - "dd0100000442"+ // field 27, encoding 5, value 33.0 - "e1010000000000005040"+ // field 28, encoding 1, value 64.0 - "e1010000000000405040"+ // field 28, encoding 1, value 65.0 - "ea0105"+"68656c6c6f"+ // field 29, encoding 2, string "hello" - "ea0106"+"7361696c6f72"+ // field 29, encoding 2, string "sailor" - "c00201"+ // field 40, encoding 0, value 1 - "c80220"+ // field 41, encoding 0, value 32 - "d00240"+ // field 42, encoding 0, value 64 - "dd0240010000"+ // field 43, encoding 5, value 320 - "e1028002000000000000"+ // field 44, encoding 1, value 640 - "e8028019"+ // field 45, encoding 0, value 3200 - "f0028032"+ // field 46, encoding 0, value 6400 - "fd02e0659948"+ // field 47, encoding 5, value 314159.0 - "81030000000050971041"+ // field 48, encoding 1, value 271828.0 - "8a0310"+"68656c6c6f2c2022776f726c6421220a"+ // field 49, encoding 2 string "hello, \"world!\"\n" - "b304"+ // start group field 70 level 1 - "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" - "b404"+ // end group field 70 level 1 - "8305"+ // start group field 80 level 1 - "8a0508"+"7265706561746564"+ // field 81, encoding 2, string "repeated" - "8405"+ // end group field 80 level 1 - "8305"+ // start group field 80 level 1 - "8a0508"+"7265706561746564"+ // field 81, encoding 2, string "repeated" - "8405"+ // end group field 80 level 1 - "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" - "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 - "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 - "ca0c03"+"626967"+ // field 201, encoding 2, string "big" - "ca0c04"+"6e6f7365"+ // field 201, encoding 2, string "nose" - "d00c40"+ // field 202, encoding 0, value 32 - "d00c3f"+ // field 202, encoding 0, value -32 - "d80c8001"+ // field 203, encoding 0, value 64 - "d80c7f"+ // field 203, encoding 0, value -64 - "8a1907"+"4269676e6f7365"+ // field 401, encoding 2, string "Bignose" - "90193f"+ // field 402, encoding 0, value 63 - "98197f") // field 403, encoding 0, value 127 - -} - -// All required fields set, all packed repeated fields given two values. -func TestEncodeDecode6(t *testing.T) { - pb := initGoTest(false) - pb.F_BoolRepeatedPacked = []bool{false, true} - pb.F_Int32RepeatedPacked = []int32{32, 33} - pb.F_Int64RepeatedPacked = []int64{64, 65} - pb.F_Fixed32RepeatedPacked = []uint32{3232, 3333} - pb.F_Fixed64RepeatedPacked = []uint64{6464, 6565} - pb.F_Uint32RepeatedPacked = []uint32{323232, 333333} - pb.F_Uint64RepeatedPacked = []uint64{646464, 656565} - pb.F_FloatRepeatedPacked = []float32{32., 33.} - pb.F_DoubleRepeatedPacked = []float64{64., 65.} - pb.F_Sint32RepeatedPacked = []int32{32, -32} - pb.F_Sint64RepeatedPacked = []int64{64, -64} - - overify(t, pb, - "0807"+ // field 1, encoding 0, value 7 - "220d"+"0a056c6162656c120474797065"+ // field 4, encoding 2 (GoTestField) - "5001"+ // field 10, encoding 0, value 1 - "5803"+ // field 11, encoding 0, value 3 - "6006"+ // field 12, encoding 0, value 6 - "6d20000000"+ // field 13, encoding 5, value 32 - "714000000000000000"+ // field 14, encoding 1, value 64 - "78a019"+ // field 15, encoding 0, value 3232 - "8001c032"+ // field 16, encoding 0, value 6464 - "8d0100004a45"+ // field 17, encoding 5, value 3232.0 - "9101000000000040b940"+ // field 18, encoding 1, value 6464.0 - "9a0106"+"737472696e67"+ // field 19, encoding 2 string "string" - "9203020001"+ // field 50, encoding 2, 2 bytes, value 0, value 1 - "9a03022021"+ // field 51, encoding 2, 2 bytes, value 32, value 33 - "a203024041"+ // field 52, encoding 2, 2 bytes, value 64, value 65 - "aa0308"+ // field 53, encoding 2, 8 bytes - "a00c0000050d0000"+ // value 3232, value 3333 - "b20310"+ // field 54, encoding 2, 16 bytes - "4019000000000000a519000000000000"+ // value 6464, value 6565 - "ba0306"+ // field 55, encoding 2, 6 bytes - "a0dd1395ac14"+ // value 323232, value 333333 - "c20306"+ // field 56, encoding 2, 6 bytes - "c0ba27b58928"+ // value 646464, value 656565 - "ca0308"+ // field 57, encoding 2, 8 bytes - "0000004200000442"+ // value 32.0, value 33.0 - "d20310"+ // field 58, encoding 2, 16 bytes - "00000000000050400000000000405040"+ // value 64.0, value 65.0 - "b304"+ // start group field 70 level 1 - "ba0408"+"7265717569726564"+ // field 71, encoding 2, string "required" - "b404"+ // end group field 70 level 1 - "aa0605"+"6279746573"+ // field 101, encoding 2 string "bytes" - "b0063f"+ // field 102, encoding 0, 0x3f zigzag32 - "b8067f"+ // field 103, encoding 0, 0x7f zigzag64 - "b21f02"+ // field 502, encoding 2, 2 bytes - "403f"+ // value 32, value -32 - "ba1f03"+ // field 503, encoding 2, 3 bytes - "80017f") // value 64, value -64 -} - -// Test that we can encode empty bytes fields. -func TestEncodeDecodeBytes1(t *testing.T) { - pb := initGoTest(false) - - // Create our bytes - pb.F_BytesRequired = []byte{} - pb.F_BytesRepeated = [][]byte{{}} - pb.F_BytesOptional = []byte{} - - d, err := Marshal(pb) - if err != nil { - t.Error(err) - } - - pbd := new(GoTest) - if err := Unmarshal(d, pbd); err != nil { - t.Error(err) - } - - if pbd.F_BytesRequired == nil || len(pbd.F_BytesRequired) != 0 { - t.Error("required empty bytes field is incorrect") - } - if pbd.F_BytesRepeated == nil || len(pbd.F_BytesRepeated) == 1 && pbd.F_BytesRepeated[0] == nil { - t.Error("repeated empty bytes field is incorrect") - } - if pbd.F_BytesOptional == nil || len(pbd.F_BytesOptional) != 0 { - t.Error("optional empty bytes field is incorrect") - } -} - -// Test that we encode nil-valued fields of a repeated bytes field correctly. -// Since entries in a repeated field cannot be nil, nil must mean empty value. -func TestEncodeDecodeBytes2(t *testing.T) { - pb := initGoTest(false) - - // Create our bytes - pb.F_BytesRepeated = [][]byte{nil} - - d, err := Marshal(pb) - if err != nil { - t.Error(err) - } - - pbd := new(GoTest) - if err := Unmarshal(d, pbd); err != nil { - t.Error(err) - } - - if len(pbd.F_BytesRepeated) != 1 || pbd.F_BytesRepeated[0] == nil { - t.Error("Unexpected value for repeated bytes field") - } -} - -// All required fields set, defaults provided, all repeated fields given two values. -func TestSkippingUnrecognizedFields(t *testing.T) { - o := old() - pb := initGoTestField() - - // Marshal it normally. - o.Marshal(pb) - - // Now new a GoSkipTest record. - skip := &GoSkipTest{ - SkipInt32: Int32(32), - SkipFixed32: Uint32(3232), - SkipFixed64: Uint64(6464), - SkipString: String("skipper"), - Skipgroup: &GoSkipTest_SkipGroup{ - GroupInt32: Int32(75), - GroupString: String("wxyz"), - }, - } - - // Marshal it into same buffer. - o.Marshal(skip) - - pbd := new(GoTestField) - o.Unmarshal(pbd) - - // The __unrecognized field should be a marshaling of GoSkipTest - skipd := new(GoSkipTest) - - o.SetBuf(pbd.XXX_unrecognized) - o.Unmarshal(skipd) - - if *skipd.SkipInt32 != *skip.SkipInt32 { - t.Error("skip int32", skipd.SkipInt32) - } - if *skipd.SkipFixed32 != *skip.SkipFixed32 { - t.Error("skip fixed32", skipd.SkipFixed32) - } - if *skipd.SkipFixed64 != *skip.SkipFixed64 { - t.Error("skip fixed64", skipd.SkipFixed64) - } - if *skipd.SkipString != *skip.SkipString { - t.Error("skip string", *skipd.SkipString) - } - if *skipd.Skipgroup.GroupInt32 != *skip.Skipgroup.GroupInt32 { - t.Error("skip group int32", skipd.Skipgroup.GroupInt32) - } - if *skipd.Skipgroup.GroupString != *skip.Skipgroup.GroupString { - t.Error("skip group string", *skipd.Skipgroup.GroupString) - } -} - -// Check that unrecognized fields of a submessage are preserved. -func TestSubmessageUnrecognizedFields(t *testing.T) { - nm := &NewMessage{ - Nested: &NewMessage_Nested{ - Name: String("Nigel"), - FoodGroup: String("carbs"), - }, - } - b, err := Marshal(nm) - if err != nil { - t.Fatalf("Marshal of NewMessage: %v", err) - } - - // Unmarshal into an OldMessage. - om := new(OldMessage) - if err := Unmarshal(b, om); err != nil { - t.Fatalf("Unmarshal to OldMessage: %v", err) - } - exp := &OldMessage{ - Nested: &OldMessage_Nested{ - Name: String("Nigel"), - // normal protocol buffer users should not do this - XXX_unrecognized: []byte("\x12\x05carbs"), - }, - } - if !Equal(om, exp) { - t.Errorf("om = %v, want %v", om, exp) - } - - // Clone the OldMessage. - om = Clone(om).(*OldMessage) - if !Equal(om, exp) { - t.Errorf("Clone(om) = %v, want %v", om, exp) - } - - // Marshal the OldMessage, then unmarshal it into an empty NewMessage. - if b, err = Marshal(om); err != nil { - t.Fatalf("Marshal of OldMessage: %v", err) - } - t.Logf("Marshal(%v) -> %q", om, b) - nm2 := new(NewMessage) - if err := Unmarshal(b, nm2); err != nil { - t.Fatalf("Unmarshal to NewMessage: %v", err) - } - if !Equal(nm, nm2) { - t.Errorf("NewMessage round-trip: %v => %v", nm, nm2) - } -} - -// Check that an int32 field can be upgraded to an int64 field. -func TestNegativeInt32(t *testing.T) { - om := &OldMessage{ - Num: Int32(-1), - } - b, err := Marshal(om) - if err != nil { - t.Fatalf("Marshal of OldMessage: %v", err) - } - - // Check the size. It should be 11 bytes; - // 1 for the field/wire type, and 10 for the negative number. - if len(b) != 11 { - t.Errorf("%v marshaled as %q, wanted 11 bytes", om, b) - } - - // Unmarshal into a NewMessage. - nm := new(NewMessage) - if err := Unmarshal(b, nm); err != nil { - t.Fatalf("Unmarshal to NewMessage: %v", err) - } - want := &NewMessage{ - Num: Int64(-1), - } - if !Equal(nm, want) { - t.Errorf("nm = %v, want %v", nm, want) - } -} - -// Check that we can grow an array (repeated field) to have many elements. -// This test doesn't depend only on our encoding; for variety, it makes sure -// we create, encode, and decode the correct contents explicitly. It's therefore -// a bit messier. -// This test also uses (and hence tests) the Marshal/Unmarshal functions -// instead of the methods. -func TestBigRepeated(t *testing.T) { - pb := initGoTest(true) - - // Create the arrays - const N = 50 // Internally the library starts much smaller. - pb.Repeatedgroup = make([]*GoTest_RepeatedGroup, N) - pb.F_Sint64Repeated = make([]int64, N) - pb.F_Sint32Repeated = make([]int32, N) - pb.F_BytesRepeated = make([][]byte, N) - pb.F_StringRepeated = make([]string, N) - pb.F_DoubleRepeated = make([]float64, N) - pb.F_FloatRepeated = make([]float32, N) - pb.F_Uint64Repeated = make([]uint64, N) - pb.F_Uint32Repeated = make([]uint32, N) - pb.F_Fixed64Repeated = make([]uint64, N) - pb.F_Fixed32Repeated = make([]uint32, N) - pb.F_Int64Repeated = make([]int64, N) - pb.F_Int32Repeated = make([]int32, N) - pb.F_BoolRepeated = make([]bool, N) - pb.RepeatedField = make([]*GoTestField, N) - - // Fill in the arrays with checkable values. - igtf := initGoTestField() - igtrg := initGoTest_RepeatedGroup() - for i := 0; i < N; i++ { - pb.Repeatedgroup[i] = igtrg - pb.F_Sint64Repeated[i] = int64(i) - pb.F_Sint32Repeated[i] = int32(i) - s := fmt.Sprint(i) - pb.F_BytesRepeated[i] = []byte(s) - pb.F_StringRepeated[i] = s - pb.F_DoubleRepeated[i] = float64(i) - pb.F_FloatRepeated[i] = float32(i) - pb.F_Uint64Repeated[i] = uint64(i) - pb.F_Uint32Repeated[i] = uint32(i) - pb.F_Fixed64Repeated[i] = uint64(i) - pb.F_Fixed32Repeated[i] = uint32(i) - pb.F_Int64Repeated[i] = int64(i) - pb.F_Int32Repeated[i] = int32(i) - pb.F_BoolRepeated[i] = i%2 == 0 - pb.RepeatedField[i] = igtf - } - - // Marshal. - buf, _ := Marshal(pb) - - // Now test Unmarshal by recreating the original buffer. - pbd := new(GoTest) - Unmarshal(buf, pbd) - - // Check the checkable values - for i := uint64(0); i < N; i++ { - if pbd.Repeatedgroup[i] == nil { // TODO: more checking? - t.Error("pbd.Repeatedgroup bad") - } - var x uint64 - x = uint64(pbd.F_Sint64Repeated[i]) - if x != i { - t.Error("pbd.F_Sint64Repeated bad", x, i) - } - x = uint64(pbd.F_Sint32Repeated[i]) - if x != i { - t.Error("pbd.F_Sint32Repeated bad", x, i) - } - s := fmt.Sprint(i) - equalbytes(pbd.F_BytesRepeated[i], []byte(s), t) - if pbd.F_StringRepeated[i] != s { - t.Error("pbd.F_Sint32Repeated bad", pbd.F_StringRepeated[i], i) - } - x = uint64(pbd.F_DoubleRepeated[i]) - if x != i { - t.Error("pbd.F_DoubleRepeated bad", x, i) - } - x = uint64(pbd.F_FloatRepeated[i]) - if x != i { - t.Error("pbd.F_FloatRepeated bad", x, i) - } - x = pbd.F_Uint64Repeated[i] - if x != i { - t.Error("pbd.F_Uint64Repeated bad", x, i) - } - x = uint64(pbd.F_Uint32Repeated[i]) - if x != i { - t.Error("pbd.F_Uint32Repeated bad", x, i) - } - x = pbd.F_Fixed64Repeated[i] - if x != i { - t.Error("pbd.F_Fixed64Repeated bad", x, i) - } - x = uint64(pbd.F_Fixed32Repeated[i]) - if x != i { - t.Error("pbd.F_Fixed32Repeated bad", x, i) - } - x = uint64(pbd.F_Int64Repeated[i]) - if x != i { - t.Error("pbd.F_Int64Repeated bad", x, i) - } - x = uint64(pbd.F_Int32Repeated[i]) - if x != i { - t.Error("pbd.F_Int32Repeated bad", x, i) - } - if pbd.F_BoolRepeated[i] != (i%2 == 0) { - t.Error("pbd.F_BoolRepeated bad", x, i) - } - if pbd.RepeatedField[i] == nil { // TODO: more checking? - t.Error("pbd.RepeatedField bad") - } - } -} - -// Verify we give a useful message when decoding to the wrong structure type. -func TestTypeMismatch(t *testing.T) { - pb1 := initGoTest(true) - - // Marshal - o := old() - o.Marshal(pb1) - - // Now Unmarshal it to the wrong type. - pb2 := initGoTestField() - err := o.Unmarshal(pb2) - if err == nil { - t.Error("expected error, got no error") - } else if !strings.Contains(err.Error(), "bad wiretype") { - t.Error("expected bad wiretype error, got", err) - } -} - -func encodeDecode(t *testing.T, in, out Message, msg string) { - buf, err := Marshal(in) - if err != nil { - t.Fatalf("failed marshaling %v: %v", msg, err) - } - if err := Unmarshal(buf, out); err != nil { - t.Fatalf("failed unmarshaling %v: %v", msg, err) - } -} - -func TestPackedNonPackedDecoderSwitching(t *testing.T) { - np, p := new(NonPackedTest), new(PackedTest) - - // non-packed -> packed - np.A = []int32{0, 1, 1, 2, 3, 5} - encodeDecode(t, np, p, "non-packed -> packed") - if !reflect.DeepEqual(np.A, p.B) { - t.Errorf("failed non-packed -> packed; np.A=%+v, p.B=%+v", np.A, p.B) - } - - // packed -> non-packed - np.Reset() - p.B = []int32{3, 1, 4, 1, 5, 9} - encodeDecode(t, p, np, "packed -> non-packed") - if !reflect.DeepEqual(p.B, np.A) { - t.Errorf("failed packed -> non-packed; p.B=%+v, np.A=%+v", p.B, np.A) - } -} - -func TestProto1RepeatedGroup(t *testing.T) { - pb := &MessageList{ - Message: []*MessageList_Message{ - { - Name: String("blah"), - Count: Int32(7), - }, - // NOTE: pb.Message[1] is a nil - nil, - }, - } - - o := old() - err := o.Marshal(pb) - if err == nil || !strings.Contains(err.Error(), "repeated field Message has nil") { - t.Fatalf("unexpected or no error when marshaling: %v", err) - } -} - -// Test that enums work. Checks for a bug introduced by making enums -// named types instead of int32: newInt32FromUint64 would crash with -// a type mismatch in reflect.PointTo. -func TestEnum(t *testing.T) { - pb := new(GoEnum) - pb.Foo = FOO_FOO1.Enum() - o := old() - if err := o.Marshal(pb); err != nil { - t.Fatal("error encoding enum:", err) - } - pb1 := new(GoEnum) - if err := o.Unmarshal(pb1); err != nil { - t.Fatal("error decoding enum:", err) - } - if *pb1.Foo != FOO_FOO1 { - t.Error("expected 7 but got ", *pb1.Foo) - } -} - -// Enum types have String methods. Check that enum fields can be printed. -// We don't care what the value actually is, just as long as it doesn't crash. -func TestPrintingNilEnumFields(t *testing.T) { - pb := new(GoEnum) - fmt.Sprintf("%+v", pb) -} - -// Verify that absent required fields cause Marshal/Unmarshal to return errors. -func TestRequiredFieldEnforcement(t *testing.T) { - pb := new(GoTestField) - _, err := Marshal(pb) - if err == nil { - t.Error("marshal: expected error, got nil") - } else if strings.Index(err.Error(), "Label") < 0 { - t.Errorf("marshal: bad error type: %v", err) - } - - // A slightly sneaky, yet valid, proto. It encodes the same required field twice, - // so simply counting the required fields is insufficient. - // field 1, encoding 2, value "hi" - buf := []byte("\x0A\x02hi\x0A\x02hi") - err = Unmarshal(buf, pb) - if err == nil { - t.Error("unmarshal: expected error, got nil") - } else if strings.Index(err.Error(), "{Unknown}") < 0 { - t.Errorf("unmarshal: bad error type: %v", err) - } -} - -func TestTypedNilMarshal(t *testing.T) { - // A typed nil should return ErrNil and not crash. - _, err := Marshal((*GoEnum)(nil)) - if err != ErrNil { - t.Errorf("Marshal: got err %v, want ErrNil", err) - } -} - -// A type that implements the Marshaler interface, but is not nillable. -type nonNillableInt uint64 - -func (nni nonNillableInt) Marshal() ([]byte, error) { - return EncodeVarint(uint64(nni)), nil -} - -type NNIMessage struct { - nni nonNillableInt -} - -func (*NNIMessage) Reset() {} -func (*NNIMessage) String() string { return "" } -func (*NNIMessage) ProtoMessage() {} - -// A type that implements the Marshaler interface and is nillable. -type nillableMessage struct { - x uint64 -} - -func (nm *nillableMessage) Marshal() ([]byte, error) { - return EncodeVarint(nm.x), nil -} - -type NMMessage struct { - nm *nillableMessage -} - -func (*NMMessage) Reset() {} -func (*NMMessage) String() string { return "" } -func (*NMMessage) ProtoMessage() {} - -// Verify a type that uses the Marshaler interface, but has a nil pointer. -func TestNilMarshaler(t *testing.T) { - // Try a struct with a Marshaler field that is nil. - // It should be directly marshable. - nmm := new(NMMessage) - if _, err := Marshal(nmm); err != nil { - t.Error("unexpected error marshaling nmm: ", err) - } - - // Try a struct with a Marshaler field that is not nillable. - nnim := new(NNIMessage) - nnim.nni = 7 - var _ Marshaler = nnim.nni // verify it is truly a Marshaler - if _, err := Marshal(nnim); err != nil { - t.Error("unexpected error marshaling nnim: ", err) - } -} - -func TestAllSetDefaults(t *testing.T) { - // Exercise SetDefaults with all scalar field types. - m := &Defaults{ - // NaN != NaN, so override that here. - F_Nan: Float32(1.7), - } - expected := &Defaults{ - F_Bool: Bool(true), - F_Int32: Int32(32), - F_Int64: Int64(64), - F_Fixed32: Uint32(320), - F_Fixed64: Uint64(640), - F_Uint32: Uint32(3200), - F_Uint64: Uint64(6400), - F_Float: Float32(314159), - F_Double: Float64(271828), - F_String: String(`hello, "world!"` + "\n"), - F_Bytes: []byte("Bignose"), - F_Sint32: Int32(-32), - F_Sint64: Int64(-64), - F_Enum: Defaults_GREEN.Enum(), - F_Pinf: Float32(float32(math.Inf(1))), - F_Ninf: Float32(float32(math.Inf(-1))), - F_Nan: Float32(1.7), - StrZero: String(""), - } - SetDefaults(m) - if !Equal(m, expected) { - t.Errorf("SetDefaults failed\n got %v\nwant %v", m, expected) - } -} - -func TestSetDefaultsWithSetField(t *testing.T) { - // Check that a set value is not overridden. - m := &Defaults{ - F_Int32: Int32(12), - } - SetDefaults(m) - if v := m.GetF_Int32(); v != 12 { - t.Errorf("m.FInt32 = %v, want 12", v) - } -} - -func TestSetDefaultsWithSubMessage(t *testing.T) { - m := &OtherMessage{ - Key: Int64(123), - Inner: &InnerMessage{ - Host: String("gopher"), - }, - } - expected := &OtherMessage{ - Key: Int64(123), - Inner: &InnerMessage{ - Host: String("gopher"), - Port: Int32(4000), - }, - } - SetDefaults(m) - if !Equal(m, expected) { - t.Errorf("\n got %v\nwant %v", m, expected) - } -} - -func TestSetDefaultsWithRepeatedSubMessage(t *testing.T) { - m := &MyMessage{ - RepInner: []*InnerMessage{{}}, - } - expected := &MyMessage{ - RepInner: []*InnerMessage{{ - Port: Int32(4000), - }}, - } - SetDefaults(m) - if !Equal(m, expected) { - t.Errorf("\n got %v\nwant %v", m, expected) - } -} - -func TestSetDefaultWithRepeatedNonMessage(t *testing.T) { - m := &MyMessage{ - Pet: []string{"turtle", "wombat"}, - } - expected := Clone(m) - SetDefaults(m) - if !Equal(m, expected) { - t.Errorf("\n got %v\nwant %v", m, expected) - } -} - -func TestMaximumTagNumber(t *testing.T) { - m := &MaxTag{ - LastField: String("natural goat essence"), - } - buf, err := Marshal(m) - if err != nil { - t.Fatalf("proto.Marshal failed: %v", err) - } - m2 := new(MaxTag) - if err := Unmarshal(buf, m2); err != nil { - t.Fatalf("proto.Unmarshal failed: %v", err) - } - if got, want := m2.GetLastField(), *m.LastField; got != want { - t.Errorf("got %q, want %q", got, want) - } -} - -func TestJSON(t *testing.T) { - m := &MyMessage{ - Count: Int32(4), - Pet: []string{"bunny", "kitty"}, - Inner: &InnerMessage{ - Host: String("cauchy"), - }, - Bikeshed: MyMessage_GREEN.Enum(), - } - const expected = `{"count":4,"pet":["bunny","kitty"],"inner":{"host":"cauchy"},"bikeshed":1}` - - b, err := json.Marshal(m) - if err != nil { - t.Fatalf("json.Marshal failed: %v", err) - } - s := string(b) - if s != expected { - t.Errorf("got %s\nwant %s", s, expected) - } - - received := new(MyMessage) - if err := json.Unmarshal(b, received); err != nil { - t.Fatalf("json.Unmarshal failed: %v", err) - } - if !Equal(received, m) { - t.Fatalf("got %s, want %s", received, m) - } - - // Test unmarshalling of JSON with symbolic enum name. - const old = `{"count":4,"pet":["bunny","kitty"],"inner":{"host":"cauchy"},"bikeshed":"GREEN"}` - received.Reset() - if err := json.Unmarshal([]byte(old), received); err != nil { - t.Fatalf("json.Unmarshal failed: %v", err) - } - if !Equal(received, m) { - t.Fatalf("got %s, want %s", received, m) - } -} - -func TestBadWireType(t *testing.T) { - b := []byte{7<<3 | 6} // field 7, wire type 6 - pb := new(OtherMessage) - if err := Unmarshal(b, pb); err == nil { - t.Errorf("Unmarshal did not fail") - } else if !strings.Contains(err.Error(), "unknown wire type") { - t.Errorf("wrong error: %v", err) - } -} - -func TestBytesWithInvalidLength(t *testing.T) { - // If a byte sequence has an invalid (negative) length, Unmarshal should not panic. - b := []byte{2<<3 | WireBytes, 0xff, 0xff, 0xff, 0xff, 0xff, 0} - Unmarshal(b, new(MyMessage)) -} - -func TestLengthOverflow(t *testing.T) { - // Overflowing a length should not panic. - b := []byte{2<<3 | WireBytes, 1, 1, 3<<3 | WireBytes, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x01} - Unmarshal(b, new(MyMessage)) -} - -func TestVarintOverflow(t *testing.T) { - // Overflowing a 64-bit length should not be allowed. - b := []byte{1<<3 | WireVarint, 0x01, 3<<3 | WireBytes, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x01} - if err := Unmarshal(b, new(MyMessage)); err == nil { - t.Fatalf("Overflowed uint64 length without error") - } -} - -func TestUnmarshalFuzz(t *testing.T) { - const N = 1000 - seed := time.Now().UnixNano() - t.Logf("RNG seed is %d", seed) - rng := rand.New(rand.NewSource(seed)) - buf := make([]byte, 20) - for i := 0; i < N; i++ { - for j := range buf { - buf[j] = byte(rng.Intn(256)) - } - fuzzUnmarshal(t, buf) - } -} - -func TestMergeMessages(t *testing.T) { - pb := &MessageList{Message: []*MessageList_Message{{Name: String("x"), Count: Int32(1)}}} - data, err := Marshal(pb) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - - pb1 := new(MessageList) - if err := Unmarshal(data, pb1); err != nil { - t.Fatalf("first Unmarshal: %v", err) - } - if err := Unmarshal(data, pb1); err != nil { - t.Fatalf("second Unmarshal: %v", err) - } - if len(pb1.Message) != 1 { - t.Errorf("two Unmarshals produced %d Messages, want 1", len(pb1.Message)) - } - - pb2 := new(MessageList) - if err := UnmarshalMerge(data, pb2); err != nil { - t.Fatalf("first UnmarshalMerge: %v", err) - } - if err := UnmarshalMerge(data, pb2); err != nil { - t.Fatalf("second UnmarshalMerge: %v", err) - } - if len(pb2.Message) != 2 { - t.Errorf("two UnmarshalMerges produced %d Messages, want 2", len(pb2.Message)) - } -} - -func TestExtensionMarshalOrder(t *testing.T) { - m := &MyMessage{Count: Int(123)} - if err := SetExtension(m, E_Ext_More, &Ext{Data: String("alpha")}); err != nil { - t.Fatalf("SetExtension: %v", err) - } - if err := SetExtension(m, E_Ext_Text, String("aleph")); err != nil { - t.Fatalf("SetExtension: %v", err) - } - if err := SetExtension(m, E_Ext_Number, Int32(1)); err != nil { - t.Fatalf("SetExtension: %v", err) - } - - // Serialize m several times, and check we get the same bytes each time. - var orig []byte - for i := 0; i < 100; i++ { - b, err := Marshal(m) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if i == 0 { - orig = b - continue - } - if !bytes.Equal(b, orig) { - t.Errorf("Bytes differ on attempt #%d", i) - } - } -} - -// Many extensions, because small maps might not iterate differently on each iteration. -var exts = []*ExtensionDesc{ - E_X201, - E_X202, - E_X203, - E_X204, - E_X205, - E_X206, - E_X207, - E_X208, - E_X209, - E_X210, - E_X211, - E_X212, - E_X213, - E_X214, - E_X215, - E_X216, - E_X217, - E_X218, - E_X219, - E_X220, - E_X221, - E_X222, - E_X223, - E_X224, - E_X225, - E_X226, - E_X227, - E_X228, - E_X229, - E_X230, - E_X231, - E_X232, - E_X233, - E_X234, - E_X235, - E_X236, - E_X237, - E_X238, - E_X239, - E_X240, - E_X241, - E_X242, - E_X243, - E_X244, - E_X245, - E_X246, - E_X247, - E_X248, - E_X249, - E_X250, -} - -func TestMessageSetMarshalOrder(t *testing.T) { - m := &MyMessageSet{} - for _, x := range exts { - if err := SetExtension(m, x, &Empty{}); err != nil { - t.Fatalf("SetExtension: %v", err) - } - } - - buf, err := Marshal(m) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - - // Serialize m several times, and check we get the same bytes each time. - for i := 0; i < 10; i++ { - b1, err := Marshal(m) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if !bytes.Equal(b1, buf) { - t.Errorf("Bytes differ on re-Marshal #%d", i) - } - - m2 := &MyMessageSet{} - if err := Unmarshal(buf, m2); err != nil { - t.Errorf("Unmarshal: %v", err) - } - b2, err := Marshal(m2) - if err != nil { - t.Errorf("re-Marshal: %v", err) - } - if !bytes.Equal(b2, buf) { - t.Errorf("Bytes differ on round-trip #%d", i) - } - } -} - -func TestUnmarshalMergesMessages(t *testing.T) { - // If a nested message occurs twice in the input, - // the fields should be merged when decoding. - a := &OtherMessage{ - Key: Int64(123), - Inner: &InnerMessage{ - Host: String("polhode"), - Port: Int32(1234), - }, - } - aData, err := Marshal(a) - if err != nil { - t.Fatalf("Marshal(a): %v", err) - } - b := &OtherMessage{ - Weight: Float32(1.2), - Inner: &InnerMessage{ - Host: String("herpolhode"), - Connected: Bool(true), - }, - } - bData, err := Marshal(b) - if err != nil { - t.Fatalf("Marshal(b): %v", err) - } - want := &OtherMessage{ - Key: Int64(123), - Weight: Float32(1.2), - Inner: &InnerMessage{ - Host: String("herpolhode"), - Port: Int32(1234), - Connected: Bool(true), - }, - } - got := new(OtherMessage) - if err := Unmarshal(append(aData, bData...), got); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if !Equal(got, want) { - t.Errorf("\n got %v\nwant %v", got, want) - } -} - -func TestEncodingSizes(t *testing.T) { - tests := []struct { - m Message - n int - }{ - {&Defaults{F_Int32: Int32(math.MaxInt32)}, 6}, - {&Defaults{F_Int32: Int32(math.MinInt32)}, 11}, - {&Defaults{F_Uint32: Uint32(uint32(math.MaxInt32) + 1)}, 6}, - {&Defaults{F_Uint32: Uint32(math.MaxUint32)}, 6}, - } - for _, test := range tests { - b, err := Marshal(test.m) - if err != nil { - t.Errorf("Marshal(%v): %v", test.m, err) - continue - } - if len(b) != test.n { - t.Errorf("Marshal(%v) yielded %d bytes, want %d bytes", test.m, len(b), test.n) - } - } -} - -func TestRequiredNotSetError(t *testing.T) { - pb := initGoTest(false) - pb.RequiredField.Label = nil - pb.F_Int32Required = nil - pb.F_Int64Required = nil - - expected := "0807" + // field 1, encoding 0, value 7 - "2206" + "120474797065" + // field 4, encoding 2 (GoTestField) - "5001" + // field 10, encoding 0, value 1 - "6d20000000" + // field 13, encoding 5, value 0x20 - "714000000000000000" + // field 14, encoding 1, value 0x40 - "78a019" + // field 15, encoding 0, value 0xca0 = 3232 - "8001c032" + // field 16, encoding 0, value 0x1940 = 6464 - "8d0100004a45" + // field 17, encoding 5, value 3232.0 - "9101000000000040b940" + // field 18, encoding 1, value 6464.0 - "9a0106" + "737472696e67" + // field 19, encoding 2, string "string" - "b304" + // field 70, encoding 3, start group - "ba0408" + "7265717569726564" + // field 71, encoding 2, string "required" - "b404" + // field 70, encoding 4, end group - "aa0605" + "6279746573" + // field 101, encoding 2, string "bytes" - "b0063f" + // field 102, encoding 0, 0x3f zigzag32 - "b8067f" // field 103, encoding 0, 0x7f zigzag64 - - o := old() - bytes, err := Marshal(pb) - if _, ok := err.(*RequiredNotSetError); !ok { - fmt.Printf("marshal-1 err = %v, want *RequiredNotSetError", err) - o.DebugPrint("", bytes) - t.Fatalf("expected = %s", expected) - } - if strings.Index(err.Error(), "RequiredField.Label") < 0 { - t.Errorf("marshal-1 wrong err msg: %v", err) - } - if !equal(bytes, expected, t) { - o.DebugPrint("neq 1", bytes) - t.Fatalf("expected = %s", expected) - } - - // Now test Unmarshal by recreating the original buffer. - pbd := new(GoTest) - err = Unmarshal(bytes, pbd) - if _, ok := err.(*RequiredNotSetError); !ok { - t.Fatalf("unmarshal err = %v, want *RequiredNotSetError", err) - o.DebugPrint("", bytes) - t.Fatalf("string = %s", expected) - } - if strings.Index(err.Error(), "RequiredField.{Unknown}") < 0 { - t.Errorf("unmarshal wrong err msg: %v", err) - } - bytes, err = Marshal(pbd) - if _, ok := err.(*RequiredNotSetError); !ok { - t.Errorf("marshal-2 err = %v, want *RequiredNotSetError", err) - o.DebugPrint("", bytes) - t.Fatalf("string = %s", expected) - } - if strings.Index(err.Error(), "RequiredField.Label") < 0 { - t.Errorf("marshal-2 wrong err msg: %v", err) - } - if !equal(bytes, expected, t) { - o.DebugPrint("neq 2", bytes) - t.Fatalf("string = %s", expected) - } -} - -func fuzzUnmarshal(t *testing.T, data []byte) { - defer func() { - if e := recover(); e != nil { - t.Errorf("These bytes caused a panic: %+v", data) - t.Logf("Stack:\n%s", debug.Stack()) - t.FailNow() - } - }() - - pb := new(MyMessage) - Unmarshal(data, pb) -} - -func TestMapFieldMarshal(t *testing.T) { - m := &MessageWithMap{ - NameMapping: map[int32]string{ - 1: "Rob", - 4: "Ian", - 8: "Dave", - }, - } - b, err := Marshal(m) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - - // b should be the concatenation of these three byte sequences in some order. - parts := []string{ - "\n\a\b\x01\x12\x03Rob", - "\n\a\b\x04\x12\x03Ian", - "\n\b\b\x08\x12\x04Dave", - } - ok := false - for i := range parts { - for j := range parts { - if j == i { - continue - } - for k := range parts { - if k == i || k == j { - continue - } - try := parts[i] + parts[j] + parts[k] - if bytes.Equal(b, []byte(try)) { - ok = true - break - } - } - } - } - if !ok { - t.Fatalf("Incorrect Marshal output.\n got %q\nwant %q (or a permutation of that)", b, parts[0]+parts[1]+parts[2]) - } - t.Logf("FYI b: %q", b) - - (new(Buffer)).DebugPrint("Dump of b", b) -} - -func TestMapFieldRoundTrips(t *testing.T) { - m := &MessageWithMap{ - NameMapping: map[int32]string{ - 1: "Rob", - 4: "Ian", - 8: "Dave", - }, - MsgMapping: map[int64]*FloatingPoint{ - 0x7001: &FloatingPoint{F: Float64(2.0)}, - }, - ByteMapping: map[bool][]byte{ - false: []byte("that's not right!"), - true: []byte("aye, 'tis true!"), - }, - } - b, err := Marshal(m) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - t.Logf("FYI b: %q", b) - m2 := new(MessageWithMap) - if err := Unmarshal(b, m2); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - for _, pair := range [][2]interface{}{ - {m.NameMapping, m2.NameMapping}, - {m.MsgMapping, m2.MsgMapping}, - {m.ByteMapping, m2.ByteMapping}, - } { - if !reflect.DeepEqual(pair[0], pair[1]) { - t.Errorf("Map did not survive a round trip.\ninitial: %v\n final: %v", pair[0], pair[1]) - } - } -} - -func TestMapFieldWithNil(t *testing.T) { - m := &MessageWithMap{ - MsgMapping: map[int64]*FloatingPoint{ - 1: nil, - }, - } - b, err := Marshal(m) - if err == nil { - t.Fatalf("Marshal of bad map should have failed, got these bytes: %v", b) - } -} - -func TestOneof(t *testing.T) { - m := &Communique{} - b, err := Marshal(m) - if err != nil { - t.Fatalf("Marshal of empty message with oneof: %v", err) - } - if len(b) != 0 { - t.Errorf("Marshal of empty message yielded too many bytes: %v", b) - } - - m = &Communique{ - Union: &Communique_Name{"Barry"}, - } - - // Round-trip. - b, err = Marshal(m) - if err != nil { - t.Fatalf("Marshal of message with oneof: %v", err) - } - if len(b) != 7 { // name tag/wire (1) + name len (1) + name (5) - t.Errorf("Incorrect marshal of message with oneof: %v", b) - } - m.Reset() - if err := Unmarshal(b, m); err != nil { - t.Fatalf("Unmarshal of message with oneof: %v", err) - } - if x, ok := m.Union.(*Communique_Name); !ok || x.Name != "Barry" { - t.Errorf("After round trip, Union = %+v", m.Union) - } - if name := m.GetName(); name != "Barry" { - t.Errorf("After round trip, GetName = %q, want %q", name, "Barry") - } - - // Let's try with a message in the oneof. - m.Union = &Communique_Msg{&Strings{StringField: String("deep deep string")}} - b, err = Marshal(m) - if err != nil { - t.Fatalf("Marshal of message with oneof set to message: %v", err) - } - if len(b) != 20 { // msg tag/wire (1) + msg len (1) + msg (1 + 1 + 16) - t.Errorf("Incorrect marshal of message with oneof set to message: %v", b) - } - m.Reset() - if err := Unmarshal(b, m); err != nil { - t.Fatalf("Unmarshal of message with oneof set to message: %v", err) - } - ss, ok := m.Union.(*Communique_Msg) - if !ok || ss.Msg.GetStringField() != "deep deep string" { - t.Errorf("After round trip with oneof set to message, Union = %+v", m.Union) - } -} - -func TestInefficientPackedBool(t *testing.T) { - // https://github.com/golang/protobuf/issues/76 - inp := []byte{ - 0x12, 0x02, // 0x12 = 2<<3|2; 2 bytes - // Usually a bool should take a single byte, - // but it is permitted to be any varint. - 0xb9, 0x30, - } - if err := Unmarshal(inp, new(MoreRepeated)); err != nil { - t.Error(err) - } -} - -// Benchmarks - -func testMsg() *GoTest { - pb := initGoTest(true) - const N = 1000 // Internally the library starts much smaller. - pb.F_Int32Repeated = make([]int32, N) - pb.F_DoubleRepeated = make([]float64, N) - for i := 0; i < N; i++ { - pb.F_Int32Repeated[i] = int32(i) - pb.F_DoubleRepeated[i] = float64(i) - } - return pb -} - -func bytesMsg() *GoTest { - pb := initGoTest(true) - buf := make([]byte, 4000) - for i := range buf { - buf[i] = byte(i) - } - pb.F_BytesDefaulted = buf - return pb -} - -func benchmarkMarshal(b *testing.B, pb Message, marshal func(Message) ([]byte, error)) { - d, _ := marshal(pb) - b.SetBytes(int64(len(d))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - marshal(pb) - } -} - -func benchmarkBufferMarshal(b *testing.B, pb Message) { - p := NewBuffer(nil) - benchmarkMarshal(b, pb, func(pb0 Message) ([]byte, error) { - p.Reset() - err := p.Marshal(pb0) - return p.Bytes(), err - }) -} - -func benchmarkSize(b *testing.B, pb Message) { - benchmarkMarshal(b, pb, func(pb0 Message) ([]byte, error) { - Size(pb) - return nil, nil - }) -} - -func newOf(pb Message) Message { - in := reflect.ValueOf(pb) - if in.IsNil() { - return pb - } - return reflect.New(in.Type().Elem()).Interface().(Message) -} - -func benchmarkUnmarshal(b *testing.B, pb Message, unmarshal func([]byte, Message) error) { - d, _ := Marshal(pb) - b.SetBytes(int64(len(d))) - pbd := newOf(pb) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - unmarshal(d, pbd) - } -} - -func benchmarkBufferUnmarshal(b *testing.B, pb Message) { - p := NewBuffer(nil) - benchmarkUnmarshal(b, pb, func(d []byte, pb0 Message) error { - p.SetBuf(d) - return p.Unmarshal(pb0) - }) -} - -// Benchmark{Marshal,BufferMarshal,Size,Unmarshal,BufferUnmarshal}{,Bytes} - -func BenchmarkMarshal(b *testing.B) { - benchmarkMarshal(b, testMsg(), Marshal) -} - -func BenchmarkBufferMarshal(b *testing.B) { - benchmarkBufferMarshal(b, testMsg()) -} - -func BenchmarkSize(b *testing.B) { - benchmarkSize(b, testMsg()) -} - -func BenchmarkUnmarshal(b *testing.B) { - benchmarkUnmarshal(b, testMsg(), Unmarshal) -} - -func BenchmarkBufferUnmarshal(b *testing.B) { - benchmarkBufferUnmarshal(b, testMsg()) -} - -func BenchmarkMarshalBytes(b *testing.B) { - benchmarkMarshal(b, bytesMsg(), Marshal) -} - -func BenchmarkBufferMarshalBytes(b *testing.B) { - benchmarkBufferMarshal(b, bytesMsg()) -} - -func BenchmarkSizeBytes(b *testing.B) { - benchmarkSize(b, bytesMsg()) -} - -func BenchmarkUnmarshalBytes(b *testing.B) { - benchmarkUnmarshal(b, bytesMsg(), Unmarshal) -} - -func BenchmarkBufferUnmarshalBytes(b *testing.B) { - benchmarkBufferUnmarshal(b, bytesMsg()) -} - -func BenchmarkUnmarshalUnrecognizedFields(b *testing.B) { - b.StopTimer() - pb := initGoTestField() - skip := &GoSkipTest{ - SkipInt32: Int32(32), - SkipFixed32: Uint32(3232), - SkipFixed64: Uint64(6464), - SkipString: String("skipper"), - Skipgroup: &GoSkipTest_SkipGroup{ - GroupInt32: Int32(75), - GroupString: String("wxyz"), - }, - } - - pbd := new(GoTestField) - p := NewBuffer(nil) - p.Marshal(pb) - p.Marshal(skip) - p2 := NewBuffer(nil) - - b.StartTimer() - for i := 0; i < b.N; i++ { - p2.SetBuf(p.Bytes()) - p2.Unmarshal(pbd) - } -} diff --git a/vendor/github.com/golang/protobuf/proto/clone.go b/vendor/github.com/golang/protobuf/proto/clone.go index e98ddec9..e392575b 100644 --- a/vendor/github.com/golang/protobuf/proto/clone.go +++ b/vendor/github.com/golang/protobuf/proto/clone.go @@ -84,9 +84,15 @@ func mergeStruct(out, in reflect.Value) { mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i]) } - if emIn, ok := in.Addr().Interface().(extendableProto); ok { - emOut := out.Addr().Interface().(extendableProto) - mergeExtension(emOut.ExtensionMap(), emIn.ExtensionMap()) + if emIn, ok := extendable(in.Addr().Interface()); ok { + emOut, _ := extendable(out.Addr().Interface()) + mIn, muIn := emIn.extensionsRead() + if mIn != nil { + mOut := emOut.extensionsWrite() + muIn.Lock() + mergeExtension(mOut, mIn) + muIn.Unlock() + } } uf := in.FieldByName("XXX_unrecognized") diff --git a/vendor/github.com/golang/protobuf/proto/clone_test.go b/vendor/github.com/golang/protobuf/proto/clone_test.go deleted file mode 100644 index 76720f18..00000000 --- a/vendor/github.com/golang/protobuf/proto/clone_test.go +++ /dev/null @@ -1,267 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2011 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "testing" - - "github.com/golang/protobuf/proto" - - proto3pb "github.com/golang/protobuf/proto/proto3_proto" - pb "github.com/golang/protobuf/proto/testdata" -) - -var cloneTestMessage = &pb.MyMessage{ - Count: proto.Int32(42), - Name: proto.String("Dave"), - Pet: []string{"bunny", "kitty", "horsey"}, - Inner: &pb.InnerMessage{ - Host: proto.String("niles"), - Port: proto.Int32(9099), - Connected: proto.Bool(true), - }, - Others: []*pb.OtherMessage{ - { - Value: []byte("some bytes"), - }, - }, - Somegroup: &pb.MyMessage_SomeGroup{ - GroupField: proto.Int32(6), - }, - RepBytes: [][]byte{[]byte("sham"), []byte("wow")}, -} - -func init() { - ext := &pb.Ext{ - Data: proto.String("extension"), - } - if err := proto.SetExtension(cloneTestMessage, pb.E_Ext_More, ext); err != nil { - panic("SetExtension: " + err.Error()) - } -} - -func TestClone(t *testing.T) { - m := proto.Clone(cloneTestMessage).(*pb.MyMessage) - if !proto.Equal(m, cloneTestMessage) { - t.Errorf("Clone(%v) = %v", cloneTestMessage, m) - } - - // Verify it was a deep copy. - *m.Inner.Port++ - if proto.Equal(m, cloneTestMessage) { - t.Error("Mutating clone changed the original") - } - // Byte fields and repeated fields should be copied. - if &m.Pet[0] == &cloneTestMessage.Pet[0] { - t.Error("Pet: repeated field not copied") - } - if &m.Others[0] == &cloneTestMessage.Others[0] { - t.Error("Others: repeated field not copied") - } - if &m.Others[0].Value[0] == &cloneTestMessage.Others[0].Value[0] { - t.Error("Others[0].Value: bytes field not copied") - } - if &m.RepBytes[0] == &cloneTestMessage.RepBytes[0] { - t.Error("RepBytes: repeated field not copied") - } - if &m.RepBytes[0][0] == &cloneTestMessage.RepBytes[0][0] { - t.Error("RepBytes[0]: bytes field not copied") - } -} - -func TestCloneNil(t *testing.T) { - var m *pb.MyMessage - if c := proto.Clone(m); !proto.Equal(m, c) { - t.Errorf("Clone(%v) = %v", m, c) - } -} - -var mergeTests = []struct { - src, dst, want proto.Message -}{ - { - src: &pb.MyMessage{ - Count: proto.Int32(42), - }, - dst: &pb.MyMessage{ - Name: proto.String("Dave"), - }, - want: &pb.MyMessage{ - Count: proto.Int32(42), - Name: proto.String("Dave"), - }, - }, - { - src: &pb.MyMessage{ - Inner: &pb.InnerMessage{ - Host: proto.String("hey"), - Connected: proto.Bool(true), - }, - Pet: []string{"horsey"}, - Others: []*pb.OtherMessage{ - { - Value: []byte("some bytes"), - }, - }, - }, - dst: &pb.MyMessage{ - Inner: &pb.InnerMessage{ - Host: proto.String("niles"), - Port: proto.Int32(9099), - }, - Pet: []string{"bunny", "kitty"}, - Others: []*pb.OtherMessage{ - { - Key: proto.Int64(31415926535), - }, - { - // Explicitly test a src=nil field - Inner: nil, - }, - }, - }, - want: &pb.MyMessage{ - Inner: &pb.InnerMessage{ - Host: proto.String("hey"), - Connected: proto.Bool(true), - Port: proto.Int32(9099), - }, - Pet: []string{"bunny", "kitty", "horsey"}, - Others: []*pb.OtherMessage{ - { - Key: proto.Int64(31415926535), - }, - {}, - { - Value: []byte("some bytes"), - }, - }, - }, - }, - { - src: &pb.MyMessage{ - RepBytes: [][]byte{[]byte("wow")}, - }, - dst: &pb.MyMessage{ - Somegroup: &pb.MyMessage_SomeGroup{ - GroupField: proto.Int32(6), - }, - RepBytes: [][]byte{[]byte("sham")}, - }, - want: &pb.MyMessage{ - Somegroup: &pb.MyMessage_SomeGroup{ - GroupField: proto.Int32(6), - }, - RepBytes: [][]byte{[]byte("sham"), []byte("wow")}, - }, - }, - // Check that a scalar bytes field replaces rather than appends. - { - src: &pb.OtherMessage{Value: []byte("foo")}, - dst: &pb.OtherMessage{Value: []byte("bar")}, - want: &pb.OtherMessage{Value: []byte("foo")}, - }, - { - src: &pb.MessageWithMap{ - NameMapping: map[int32]string{6: "Nigel"}, - MsgMapping: map[int64]*pb.FloatingPoint{ - 0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)}, - }, - ByteMapping: map[bool][]byte{true: []byte("wowsa")}, - }, - dst: &pb.MessageWithMap{ - NameMapping: map[int32]string{ - 6: "Bruce", // should be overwritten - 7: "Andrew", - }, - }, - want: &pb.MessageWithMap{ - NameMapping: map[int32]string{ - 6: "Nigel", - 7: "Andrew", - }, - MsgMapping: map[int64]*pb.FloatingPoint{ - 0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)}, - }, - ByteMapping: map[bool][]byte{true: []byte("wowsa")}, - }, - }, - // proto3 shouldn't merge zero values, - // in the same way that proto2 shouldn't merge nils. - { - src: &proto3pb.Message{ - Name: "Aaron", - Data: []byte(""), // zero value, but not nil - }, - dst: &proto3pb.Message{ - HeightInCm: 176, - Data: []byte("texas!"), - }, - want: &proto3pb.Message{ - Name: "Aaron", - HeightInCm: 176, - Data: []byte("texas!"), - }, - }, - // Oneof fields should merge by assignment. - { - src: &pb.Communique{ - Union: &pb.Communique_Number{41}, - }, - dst: &pb.Communique{ - Union: &pb.Communique_Name{"Bobby Tables"}, - }, - want: &pb.Communique{ - Union: &pb.Communique_Number{41}, - }, - }, - // Oneof nil is the same as not set. - { - src: &pb.Communique{}, - dst: &pb.Communique{ - Union: &pb.Communique_Name{"Bobby Tables"}, - }, - want: &pb.Communique{ - Union: &pb.Communique_Name{"Bobby Tables"}, - }, - }, -} - -func TestMerge(t *testing.T) { - for _, m := range mergeTests { - got := proto.Clone(m.dst) - proto.Merge(got, m.src) - if !proto.Equal(got, m.want) { - t.Errorf("Merge(%v, %v)\n got %v\nwant %v\n", m.dst, m.src, got, m.want) - } - } -} diff --git a/vendor/github.com/golang/protobuf/proto/decode.go b/vendor/github.com/golang/protobuf/proto/decode.go index 5810782f..aa207298 100644 --- a/vendor/github.com/golang/protobuf/proto/decode.go +++ b/vendor/github.com/golang/protobuf/proto/decode.go @@ -61,7 +61,6 @@ var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for // int32, int64, uint32, uint64, bool, and enum // protocol buffer types. func DecodeVarint(buf []byte) (x uint64, n int) { - // x, n already 0 for shift := uint(0); shift < 64; shift += 7 { if n >= len(buf) { return 0, 0 @@ -78,13 +77,7 @@ func DecodeVarint(buf []byte) (x uint64, n int) { return 0, 0 } -// DecodeVarint reads a varint-encoded integer from the Buffer. -// This is the format for the -// int32, int64, uint32, uint64, bool, and enum -// protocol buffer types. -func (p *Buffer) DecodeVarint() (x uint64, err error) { - // x, err already 0 - +func (p *Buffer) decodeVarintSlow() (x uint64, err error) { i := p.index l := len(p.buf) @@ -107,6 +100,107 @@ func (p *Buffer) DecodeVarint() (x uint64, err error) { return } +// DecodeVarint reads a varint-encoded integer from the Buffer. +// This is the format for the +// int32, int64, uint32, uint64, bool, and enum +// protocol buffer types. +func (p *Buffer) DecodeVarint() (x uint64, err error) { + i := p.index + buf := p.buf + + if i >= len(buf) { + return 0, io.ErrUnexpectedEOF + } else if buf[i] < 0x80 { + p.index++ + return uint64(buf[i]), nil + } else if len(buf)-i < 10 { + return p.decodeVarintSlow() + } + + var b uint64 + // we already checked the first byte + x = uint64(buf[i]) - 0x80 + i++ + + b = uint64(buf[i]) + i++ + x += b << 7 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 7 + + b = uint64(buf[i]) + i++ + x += b << 14 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 14 + + b = uint64(buf[i]) + i++ + x += b << 21 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 21 + + b = uint64(buf[i]) + i++ + x += b << 28 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 28 + + b = uint64(buf[i]) + i++ + x += b << 35 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 35 + + b = uint64(buf[i]) + i++ + x += b << 42 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 42 + + b = uint64(buf[i]) + i++ + x += b << 49 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 49 + + b = uint64(buf[i]) + i++ + x += b << 56 + if b&0x80 == 0 { + goto done + } + x -= 0x80 << 56 + + b = uint64(buf[i]) + i++ + x += b << 63 + if b&0x80 == 0 { + goto done + } + // x -= 0x80 << 63 // Always zero. + + return 0, errOverflow + +done: + p.index = i + return x, nil +} + // DecodeFixed64 reads a 64-bit integer from the Buffer. // This is the format for the // fixed64, sfixed64, and double protocol buffer types. @@ -340,6 +434,8 @@ func (p *Buffer) DecodeGroup(pb Message) error { // Buffer and places the decoded result in pb. If the struct // underlying pb does not match the data in the buffer, the results can be // unpredictable. +// +// Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal. func (p *Buffer) Unmarshal(pb Message) error { // If the object can unmarshal itself, let it. if u, ok := pb.(Unmarshaler); ok { @@ -378,6 +474,11 @@ func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group wire := int(u & 0x7) if wire == WireEndGroup { if is_group { + if required > 0 { + // Not enough information to determine the exact field. + // (See below.) + return &RequiredNotSetError{"{Unknown}"} + } return nil // input is satisfied } return fmt.Errorf("proto: %s: wiretype end group for non-group", st) @@ -390,11 +491,12 @@ func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group if !ok { // Maybe it's an extension? if prop.extendable { - if e := structPointer_Interface(base, st).(extendableProto); isExtensionField(e, int32(tag)) { + if e, _ := extendable(structPointer_Interface(base, st)); isExtensionField(e, int32(tag)) { if err = o.skip(st, tag, wire); err == nil { - ext := e.ExtensionMap()[int32(tag)] // may be missing + extmap := e.extensionsWrite() + ext := extmap[int32(tag)] // may be missing ext.enc = append(ext.enc, o.buf[oi:o.index]...) - e.ExtensionMap()[int32(tag)] = ext + extmap[int32(tag)] = ext } continue } @@ -768,10 +870,11 @@ func (o *Buffer) dec_new_map(p *Properties, base structPointer) error { } } keyelem, valelem := keyptr.Elem(), valptr.Elem() - if !keyelem.IsValid() || !valelem.IsValid() { - // We did not decode the key or the value in the map entry. - // Either way, it's an invalid map entry. - return fmt.Errorf("proto: bad map data: missing key/val") + if !keyelem.IsValid() { + keyelem = reflect.Zero(p.mtype.Key()) + } + if !valelem.IsValid() { + valelem = reflect.Zero(p.mtype.Elem()) } v.SetMapIndex(keyelem, valelem) diff --git a/vendor/github.com/golang/protobuf/proto/encode.go b/vendor/github.com/golang/protobuf/proto/encode.go index 231b0740..2b30f846 100644 --- a/vendor/github.com/golang/protobuf/proto/encode.go +++ b/vendor/github.com/golang/protobuf/proto/encode.go @@ -64,8 +64,16 @@ var ( // a struct with a repeated field containing a nil element. errRepeatedHasNil = errors.New("proto: repeated field has nil element") + // errOneofHasNil is the error returned if Marshal is called with + // a struct with a oneof field containing a nil element. + errOneofHasNil = errors.New("proto: oneof field has nil value") + // ErrNil is the error returned if Marshal is called with nil. ErrNil = errors.New("proto: Marshal called with nil") + + // ErrTooLarge is the error returned if Marshal is called with a + // message that encodes to >2GB. + ErrTooLarge = errors.New("proto: message encodes to over 2 GB") ) // The fundamental encoders that put bytes on the wire. @@ -74,6 +82,10 @@ var ( const maxVarintBytes = 10 // maximum length of a varint +// maxMarshalSize is the largest allowed size of an encoded protobuf, +// since C++ and Java use signed int32s for the size. +const maxMarshalSize = 1<<31 - 1 + // EncodeVarint returns the varint encoding of x. // This is the format for the // int32, int64, uint32, uint64, bool, and enum @@ -222,10 +234,6 @@ func Marshal(pb Message) ([]byte, error) { } p := NewBuffer(nil) err := p.Marshal(pb) - var state errorState - if err != nil && !state.shouldContinue(err, nil) { - return nil, err - } if p.buf == nil && err == nil { // Return a non-nil slice on success. return []byte{}, nil @@ -254,11 +262,8 @@ func (p *Buffer) Marshal(pb Message) error { // Can the object marshal itself? if m, ok := pb.(Marshaler); ok { data, err := m.Marshal() - if err != nil { - return err - } p.buf = append(p.buf, data...) - return nil + return err } t, base, err := getbase(pb) @@ -270,9 +275,12 @@ func (p *Buffer) Marshal(pb Message) error { } if collectStats { - stats.Encode++ + (stats).Encode++ // Parens are to work around a goimports bug. } + if len(p.buf) > maxMarshalSize { + return ErrTooLarge + } return err } @@ -294,7 +302,7 @@ func Size(pb Message) (n int) { } if collectStats { - stats.Size++ + (stats).Size++ // Parens are to work around a goimports bug. } return @@ -999,7 +1007,6 @@ func size_slice_struct_message(p *Properties, base structPointer) (n int) { if p.isMarshaler { m := structPointer_Interface(structp, p.stype).(Marshaler) data, _ := m.Marshal() - n += len(p.tagcode) n += sizeRawBytes(data) continue } @@ -1058,10 +1065,32 @@ func size_slice_struct_group(p *Properties, base structPointer) (n int) { // Encode an extension map. func (o *Buffer) enc_map(p *Properties, base structPointer) error { - v := *structPointer_ExtMap(base, p.field) - if err := encodeExtensionMap(v); err != nil { + exts := structPointer_ExtMap(base, p.field) + if err := encodeExtensionsMap(*exts); err != nil { return err } + + return o.enc_map_body(*exts) +} + +func (o *Buffer) enc_exts(p *Properties, base structPointer) error { + exts := structPointer_Extensions(base, p.field) + + v, mu := exts.extensionsRead() + if v == nil { + return nil + } + + mu.Lock() + defer mu.Unlock() + if err := encodeExtensionsMap(v); err != nil { + return err + } + + return o.enc_map_body(v) +} + +func (o *Buffer) enc_map_body(v map[int32]Extension) error { // Fast-path for common cases: zero or one extensions. if len(v) <= 1 { for _, e := range v { @@ -1084,8 +1113,13 @@ func (o *Buffer) enc_map(p *Properties, base structPointer) error { } func size_map(p *Properties, base structPointer) int { - v := *structPointer_ExtMap(base, p.field) - return sizeExtensionMap(v) + v := structPointer_ExtMap(base, p.field) + return extensionsMapSize(*v) +} + +func size_exts(p *Properties, base structPointer) int { + v := structPointer_Extensions(base, p.field) + return extensionsSize(v) } // Encode a map field. @@ -1114,7 +1148,7 @@ func (o *Buffer) enc_new_map(p *Properties, base structPointer) error { if err := p.mkeyprop.enc(o, p.mkeyprop, keybase); err != nil { return err } - if err := p.mvalprop.enc(o, p.mvalprop, valbase); err != nil { + if err := p.mvalprop.enc(o, p.mvalprop, valbase); err != nil && err != ErrNil { return err } return nil @@ -1124,11 +1158,6 @@ func (o *Buffer) enc_new_map(p *Properties, base structPointer) error { for _, key := range v.MapKeys() { val := v.MapIndex(key) - // The only illegal map entry values are nil message pointers. - if val.Kind() == reflect.Ptr && val.IsNil() { - return errors.New("proto: map has nil element") - } - keycopy.Set(key) valcopy.Set(val) @@ -1216,13 +1245,18 @@ func (o *Buffer) enc_struct(prop *StructProperties, base structPointer) error { return err } } + if len(o.buf) > maxMarshalSize { + return ErrTooLarge + } } } // Do oneof fields. if prop.oneofMarshaler != nil { m := structPointer_Interface(base, prop.stype).(Message) - if err := prop.oneofMarshaler(m, o); err != nil { + if err := prop.oneofMarshaler(m, o); err == ErrNil { + return errOneofHasNil + } else if err != nil { return err } } @@ -1230,6 +1264,9 @@ func (o *Buffer) enc_struct(prop *StructProperties, base structPointer) error { // Add unrecognized fields at the end. if prop.unrecField.IsValid() { v := *structPointer_Bytes(base, prop.unrecField) + if len(o.buf)+len(v) > maxMarshalSize { + return ErrTooLarge + } if len(v) > 0 { o.buf = append(o.buf, v...) } diff --git a/vendor/github.com/golang/protobuf/proto/equal.go b/vendor/github.com/golang/protobuf/proto/equal.go index f5db1def..2ed1cf59 100644 --- a/vendor/github.com/golang/protobuf/proto/equal.go +++ b/vendor/github.com/golang/protobuf/proto/equal.go @@ -54,13 +54,17 @@ Equality is defined in this way: in a proto3 .proto file, fields are not "set"; specifically, zero length proto3 "bytes" fields are equal (nil == {}). - Two repeated fields are equal iff their lengths are the same, - and their corresponding elements are equal (a "bytes" field, - although represented by []byte, is not a repeated field) + and their corresponding elements are equal. Note a "bytes" field, + although represented by []byte, is not a repeated field and the + rule for the scalar fields described above applies. - Two unset fields are equal. - Two unknown field sets are equal if their current encoded state is equal. - Two extension sets are equal iff they have corresponding elements that are pairwise equal. + - Two map fields are equal iff their lengths are the same, + and they contain the same set of elements. Zero-length map + fields are equal. - Every other combination of things are not equal. The return value is undefined if a and b are not protocol buffers. @@ -121,9 +125,16 @@ func equalStruct(v1, v2 reflect.Value) bool { } } + if em1 := v1.FieldByName("XXX_InternalExtensions"); em1.IsValid() { + em2 := v2.FieldByName("XXX_InternalExtensions") + if !equalExtensions(v1.Type(), em1.Interface().(XXX_InternalExtensions), em2.Interface().(XXX_InternalExtensions)) { + return false + } + } + if em1 := v1.FieldByName("XXX_extensions"); em1.IsValid() { em2 := v2.FieldByName("XXX_extensions") - if !equalExtensions(v1.Type(), em1.Interface().(map[int32]Extension), em2.Interface().(map[int32]Extension)) { + if !equalExtMap(v1.Type(), em1.Interface().(map[int32]Extension), em2.Interface().(map[int32]Extension)) { return false } } @@ -184,6 +195,13 @@ func equalAny(v1, v2 reflect.Value, prop *Properties) bool { } return true case reflect.Ptr: + // Maps may have nil values in them, so check for nil. + if v1.IsNil() && v2.IsNil() { + return true + } + if v1.IsNil() != v2.IsNil() { + return false + } return equalAny(v1.Elem(), v2.Elem(), prop) case reflect.Slice: if v1.Type().Elem().Kind() == reflect.Uint8 { @@ -223,8 +241,14 @@ func equalAny(v1, v2 reflect.Value, prop *Properties) bool { } // base is the struct type that the extensions are based on. -// em1 and em2 are extension maps. -func equalExtensions(base reflect.Type, em1, em2 map[int32]Extension) bool { +// x1 and x2 are InternalExtensions. +func equalExtensions(base reflect.Type, x1, x2 XXX_InternalExtensions) bool { + em1, _ := x1.extensionsRead() + em2, _ := x2.extensionsRead() + return equalExtMap(base, em1, em2) +} + +func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool { if len(em1) != len(em2) { return false } diff --git a/vendor/github.com/golang/protobuf/proto/equal_test.go b/vendor/github.com/golang/protobuf/proto/equal_test.go deleted file mode 100644 index 7b45eaa6..00000000 --- a/vendor/github.com/golang/protobuf/proto/equal_test.go +++ /dev/null @@ -1,212 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2011 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "testing" - - . "github.com/golang/protobuf/proto" - proto3pb "github.com/golang/protobuf/proto/proto3_proto" - pb "github.com/golang/protobuf/proto/testdata" -) - -// Four identical base messages. -// The init function adds extensions to some of them. -var messageWithoutExtension = &pb.MyMessage{Count: Int32(7)} -var messageWithExtension1a = &pb.MyMessage{Count: Int32(7)} -var messageWithExtension1b = &pb.MyMessage{Count: Int32(7)} -var messageWithExtension2 = &pb.MyMessage{Count: Int32(7)} - -// Two messages with non-message extensions. -var messageWithInt32Extension1 = &pb.MyMessage{Count: Int32(8)} -var messageWithInt32Extension2 = &pb.MyMessage{Count: Int32(8)} - -func init() { - ext1 := &pb.Ext{Data: String("Kirk")} - ext2 := &pb.Ext{Data: String("Picard")} - - // messageWithExtension1a has ext1, but never marshals it. - if err := SetExtension(messageWithExtension1a, pb.E_Ext_More, ext1); err != nil { - panic("SetExtension on 1a failed: " + err.Error()) - } - - // messageWithExtension1b is the unmarshaled form of messageWithExtension1a. - if err := SetExtension(messageWithExtension1b, pb.E_Ext_More, ext1); err != nil { - panic("SetExtension on 1b failed: " + err.Error()) - } - buf, err := Marshal(messageWithExtension1b) - if err != nil { - panic("Marshal of 1b failed: " + err.Error()) - } - messageWithExtension1b.Reset() - if err := Unmarshal(buf, messageWithExtension1b); err != nil { - panic("Unmarshal of 1b failed: " + err.Error()) - } - - // messageWithExtension2 has ext2. - if err := SetExtension(messageWithExtension2, pb.E_Ext_More, ext2); err != nil { - panic("SetExtension on 2 failed: " + err.Error()) - } - - if err := SetExtension(messageWithInt32Extension1, pb.E_Ext_Number, Int32(23)); err != nil { - panic("SetExtension on Int32-1 failed: " + err.Error()) - } - if err := SetExtension(messageWithInt32Extension1, pb.E_Ext_Number, Int32(24)); err != nil { - panic("SetExtension on Int32-2 failed: " + err.Error()) - } -} - -var EqualTests = []struct { - desc string - a, b Message - exp bool -}{ - {"different types", &pb.GoEnum{}, &pb.GoTestField{}, false}, - {"equal empty", &pb.GoEnum{}, &pb.GoEnum{}, true}, - {"nil vs nil", nil, nil, true}, - {"typed nil vs typed nil", (*pb.GoEnum)(nil), (*pb.GoEnum)(nil), true}, - {"typed nil vs empty", (*pb.GoEnum)(nil), &pb.GoEnum{}, false}, - {"different typed nil", (*pb.GoEnum)(nil), (*pb.GoTestField)(nil), false}, - - {"one set field, one unset field", &pb.GoTestField{Label: String("foo")}, &pb.GoTestField{}, false}, - {"one set field zero, one unset field", &pb.GoTest{Param: Int32(0)}, &pb.GoTest{}, false}, - {"different set fields", &pb.GoTestField{Label: String("foo")}, &pb.GoTestField{Label: String("bar")}, false}, - {"equal set", &pb.GoTestField{Label: String("foo")}, &pb.GoTestField{Label: String("foo")}, true}, - - {"repeated, one set", &pb.GoTest{F_Int32Repeated: []int32{2, 3}}, &pb.GoTest{}, false}, - {"repeated, different length", &pb.GoTest{F_Int32Repeated: []int32{2, 3}}, &pb.GoTest{F_Int32Repeated: []int32{2}}, false}, - {"repeated, different value", &pb.GoTest{F_Int32Repeated: []int32{2}}, &pb.GoTest{F_Int32Repeated: []int32{3}}, false}, - {"repeated, equal", &pb.GoTest{F_Int32Repeated: []int32{2, 4}}, &pb.GoTest{F_Int32Repeated: []int32{2, 4}}, true}, - {"repeated, nil equal nil", &pb.GoTest{F_Int32Repeated: nil}, &pb.GoTest{F_Int32Repeated: nil}, true}, - {"repeated, nil equal empty", &pb.GoTest{F_Int32Repeated: nil}, &pb.GoTest{F_Int32Repeated: []int32{}}, true}, - {"repeated, empty equal nil", &pb.GoTest{F_Int32Repeated: []int32{}}, &pb.GoTest{F_Int32Repeated: nil}, true}, - - { - "nested, different", - &pb.GoTest{RequiredField: &pb.GoTestField{Label: String("foo")}}, - &pb.GoTest{RequiredField: &pb.GoTestField{Label: String("bar")}}, - false, - }, - { - "nested, equal", - &pb.GoTest{RequiredField: &pb.GoTestField{Label: String("wow")}}, - &pb.GoTest{RequiredField: &pb.GoTestField{Label: String("wow")}}, - true, - }, - - {"bytes", &pb.OtherMessage{Value: []byte("foo")}, &pb.OtherMessage{Value: []byte("foo")}, true}, - {"bytes, empty", &pb.OtherMessage{Value: []byte{}}, &pb.OtherMessage{Value: []byte{}}, true}, - {"bytes, empty vs nil", &pb.OtherMessage{Value: []byte{}}, &pb.OtherMessage{Value: nil}, false}, - { - "repeated bytes", - &pb.MyMessage{RepBytes: [][]byte{[]byte("sham"), []byte("wow")}}, - &pb.MyMessage{RepBytes: [][]byte{[]byte("sham"), []byte("wow")}}, - true, - }, - // In proto3, []byte{} and []byte(nil) are equal. - {"proto3 bytes, empty vs nil", &proto3pb.Message{Data: []byte{}}, &proto3pb.Message{Data: nil}, true}, - - {"extension vs. no extension", messageWithoutExtension, messageWithExtension1a, false}, - {"extension vs. same extension", messageWithExtension1a, messageWithExtension1b, true}, - {"extension vs. different extension", messageWithExtension1a, messageWithExtension2, false}, - - {"int32 extension vs. itself", messageWithInt32Extension1, messageWithInt32Extension1, true}, - {"int32 extension vs. a different int32", messageWithInt32Extension1, messageWithInt32Extension2, false}, - - { - "message with group", - &pb.MyMessage{ - Count: Int32(1), - Somegroup: &pb.MyMessage_SomeGroup{ - GroupField: Int32(5), - }, - }, - &pb.MyMessage{ - Count: Int32(1), - Somegroup: &pb.MyMessage_SomeGroup{ - GroupField: Int32(5), - }, - }, - true, - }, - - { - "map same", - &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, - &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, - true, - }, - { - "map different entry", - &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, - &pb.MessageWithMap{NameMapping: map[int32]string{2: "Rob"}}, - false, - }, - { - "map different key only", - &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, - &pb.MessageWithMap{NameMapping: map[int32]string{2: "Ken"}}, - false, - }, - { - "map different value only", - &pb.MessageWithMap{NameMapping: map[int32]string{1: "Ken"}}, - &pb.MessageWithMap{NameMapping: map[int32]string{1: "Rob"}}, - false, - }, - { - "oneof same", - &pb.Communique{Union: &pb.Communique_Number{41}}, - &pb.Communique{Union: &pb.Communique_Number{41}}, - true, - }, - { - "oneof one nil", - &pb.Communique{Union: &pb.Communique_Number{41}}, - &pb.Communique{}, - false, - }, - { - "oneof different", - &pb.Communique{Union: &pb.Communique_Number{41}}, - &pb.Communique{Union: &pb.Communique_Name{"Bobby Tables"}}, - false, - }, -} - -func TestEqual(t *testing.T) { - for _, tc := range EqualTests { - if res := Equal(tc.a, tc.b); res != tc.exp { - t.Errorf("%v: Equal(%v, %v) = %v, want %v", tc.desc, tc.a, tc.b, res, tc.exp) - } - } -} diff --git a/vendor/github.com/golang/protobuf/proto/extensions.go b/vendor/github.com/golang/protobuf/proto/extensions.go index 054f4f1d..eaad2183 100644 --- a/vendor/github.com/golang/protobuf/proto/extensions.go +++ b/vendor/github.com/golang/protobuf/proto/extensions.go @@ -52,14 +52,99 @@ type ExtensionRange struct { Start, End int32 // both inclusive } -// extendableProto is an interface implemented by any protocol buffer that may be extended. +// extendableProto is an interface implemented by any protocol buffer generated by the current +// proto compiler that may be extended. type extendableProto interface { + Message + ExtensionRangeArray() []ExtensionRange + extensionsWrite() map[int32]Extension + extensionsRead() (map[int32]Extension, sync.Locker) +} + +// extendableProtoV1 is an interface implemented by a protocol buffer generated by the previous +// version of the proto compiler that may be extended. +type extendableProtoV1 interface { Message ExtensionRangeArray() []ExtensionRange ExtensionMap() map[int32]Extension } +// extensionAdapter is a wrapper around extendableProtoV1 that implements extendableProto. +type extensionAdapter struct { + extendableProtoV1 +} + +func (e extensionAdapter) extensionsWrite() map[int32]Extension { + return e.ExtensionMap() +} + +func (e extensionAdapter) extensionsRead() (map[int32]Extension, sync.Locker) { + return e.ExtensionMap(), notLocker{} +} + +// notLocker is a sync.Locker whose Lock and Unlock methods are nops. +type notLocker struct{} + +func (n notLocker) Lock() {} +func (n notLocker) Unlock() {} + +// extendable returns the extendableProto interface for the given generated proto message. +// If the proto message has the old extension format, it returns a wrapper that implements +// the extendableProto interface. +func extendable(p interface{}) (extendableProto, bool) { + if ep, ok := p.(extendableProto); ok { + return ep, ok + } + if ep, ok := p.(extendableProtoV1); ok { + return extensionAdapter{ep}, ok + } + return nil, false +} + +// XXX_InternalExtensions is an internal representation of proto extensions. +// +// Each generated message struct type embeds an anonymous XXX_InternalExtensions field, +// thus gaining the unexported 'extensions' method, which can be called only from the proto package. +// +// The methods of XXX_InternalExtensions are not concurrency safe in general, +// but calls to logically read-only methods such as has and get may be executed concurrently. +type XXX_InternalExtensions struct { + // The struct must be indirect so that if a user inadvertently copies a + // generated message and its embedded XXX_InternalExtensions, they + // avoid the mayhem of a copied mutex. + // + // The mutex serializes all logically read-only operations to p.extensionMap. + // It is up to the client to ensure that write operations to p.extensionMap are + // mutually exclusive with other accesses. + p *struct { + mu sync.Mutex + extensionMap map[int32]Extension + } +} + +// extensionsWrite returns the extension map, creating it on first use. +func (e *XXX_InternalExtensions) extensionsWrite() map[int32]Extension { + if e.p == nil { + e.p = new(struct { + mu sync.Mutex + extensionMap map[int32]Extension + }) + e.p.extensionMap = make(map[int32]Extension) + } + return e.p.extensionMap +} + +// extensionsRead returns the extensions map for read-only use. It may be nil. +// The caller must hold the returned mutex's lock when accessing Elements within the map. +func (e *XXX_InternalExtensions) extensionsRead() (map[int32]Extension, sync.Locker) { + if e.p == nil { + return nil, nil + } + return e.p.extensionMap, &e.p.mu +} + var extendableProtoType = reflect.TypeOf((*extendableProto)(nil)).Elem() +var extendableProtoV1Type = reflect.TypeOf((*extendableProtoV1)(nil)).Elem() // ExtensionDesc represents an extension specification. // Used in generated code from the protocol compiler. @@ -69,6 +154,7 @@ type ExtensionDesc struct { Field int32 // field number Name string // fully-qualified name of extension, for text formatting Tag string // protobuf tag style + Filename string // name of the file in which the extension is defined } func (ed *ExtensionDesc) repeated() bool { @@ -92,8 +178,13 @@ type Extension struct { } // SetRawExtension is for testing only. -func SetRawExtension(base extendableProto, id int32, b []byte) { - base.ExtensionMap()[id] = Extension{enc: b} +func SetRawExtension(base Message, id int32, b []byte) { + epb, ok := extendable(base) + if !ok { + return + } + extmap := epb.extensionsWrite() + extmap[id] = Extension{enc: b} } // isExtensionField returns true iff the given field number is in an extension range. @@ -108,8 +199,12 @@ func isExtensionField(pb extendableProto, field int32) bool { // checkExtensionTypes checks that the given extension is valid for pb. func checkExtensionTypes(pb extendableProto, extension *ExtensionDesc) error { + var pbi interface{} = pb // Check the extended type. - if a, b := reflect.TypeOf(pb), reflect.TypeOf(extension.ExtendedType); a != b { + if ea, ok := pbi.(extensionAdapter); ok { + pbi = ea.extendableProtoV1 + } + if a, b := reflect.TypeOf(pbi), reflect.TypeOf(extension.ExtendedType); a != b { return errors.New("proto: bad extended type; " + b.String() + " does not extend " + a.String()) } // Check the range. @@ -155,8 +250,19 @@ func extensionProperties(ed *ExtensionDesc) *Properties { return prop } -// encodeExtensionMap encodes any unmarshaled (unencoded) extensions in m. -func encodeExtensionMap(m map[int32]Extension) error { +// encode encodes any unmarshaled (unencoded) extensions in e. +func encodeExtensions(e *XXX_InternalExtensions) error { + m, mu := e.extensionsRead() + if m == nil { + return nil // fast path + } + mu.Lock() + defer mu.Unlock() + return encodeExtensionsMap(m) +} + +// encode encodes any unmarshaled (unencoded) extensions in e. +func encodeExtensionsMap(m map[int32]Extension) error { for k, e := range m { if e.value == nil || e.desc == nil { // Extension is only in its encoded form. @@ -184,7 +290,17 @@ func encodeExtensionMap(m map[int32]Extension) error { return nil } -func sizeExtensionMap(m map[int32]Extension) (n int) { +func extensionsSize(e *XXX_InternalExtensions) (n int) { + m, mu := e.extensionsRead() + if m == nil { + return 0 + } + mu.Lock() + defer mu.Unlock() + return extensionsMapSize(m) +} + +func extensionsMapSize(m map[int32]Extension) (n int) { for _, e := range m { if e.value == nil || e.desc == nil { // Extension is only in its encoded form. @@ -209,26 +325,51 @@ func sizeExtensionMap(m map[int32]Extension) (n int) { } // HasExtension returns whether the given extension is present in pb. -func HasExtension(pb extendableProto, extension *ExtensionDesc) bool { +func HasExtension(pb Message, extension *ExtensionDesc) bool { // TODO: Check types, field numbers, etc.? - _, ok := pb.ExtensionMap()[extension.Field] + epb, ok := extendable(pb) + if !ok { + return false + } + extmap, mu := epb.extensionsRead() + if extmap == nil { + return false + } + mu.Lock() + _, ok = extmap[extension.Field] + mu.Unlock() return ok } // ClearExtension removes the given extension from pb. -func ClearExtension(pb extendableProto, extension *ExtensionDesc) { +func ClearExtension(pb Message, extension *ExtensionDesc) { + epb, ok := extendable(pb) + if !ok { + return + } // TODO: Check types, field numbers, etc.? - delete(pb.ExtensionMap(), extension.Field) + extmap := epb.extensionsWrite() + delete(extmap, extension.Field) } // GetExtension parses and returns the given extension of pb. // If the extension is not present and has no default value it returns ErrMissingExtension. -func GetExtension(pb extendableProto, extension *ExtensionDesc) (interface{}, error) { - if err := checkExtensionTypes(pb, extension); err != nil { +func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { + epb, ok := extendable(pb) + if !ok { + return nil, errors.New("proto: not an extendable proto") + } + + if err := checkExtensionTypes(epb, extension); err != nil { return nil, err } - emap := pb.ExtensionMap() + emap, mu := epb.extensionsRead() + if emap == nil { + return defaultExtensionValue(extension) + } + mu.Lock() + defer mu.Unlock() e, ok := emap[extension.Field] if !ok { // defaultExtensionValue returns the default value or @@ -332,10 +473,9 @@ func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) { // GetExtensions returns a slice of the extensions present in pb that are also listed in es. // The returned slice has the same length as es; missing extensions will appear as nil elements. func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, err error) { - epb, ok := pb.(extendableProto) + epb, ok := extendable(pb) if !ok { - err = errors.New("proto: not an extendable proto") - return + return nil, errors.New("proto: not an extendable proto") } extensions = make([]interface{}, len(es)) for i, e := range es { @@ -350,9 +490,44 @@ func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, e return } +// ExtensionDescs returns a new slice containing pb's extension descriptors, in undefined order. +// For non-registered extensions, ExtensionDescs returns an incomplete descriptor containing +// just the Field field, which defines the extension's field number. +func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { + epb, ok := extendable(pb) + if !ok { + return nil, fmt.Errorf("proto: %T is not an extendable proto.Message", pb) + } + registeredExtensions := RegisteredExtensions(pb) + + emap, mu := epb.extensionsRead() + if emap == nil { + return nil, nil + } + mu.Lock() + defer mu.Unlock() + extensions := make([]*ExtensionDesc, 0, len(emap)) + for extid, e := range emap { + desc := e.desc + if desc == nil { + desc = registeredExtensions[extid] + if desc == nil { + desc = &ExtensionDesc{Field: extid} + } + } + + extensions = append(extensions, desc) + } + return extensions, nil +} + // SetExtension sets the specified extension of pb to the specified value. -func SetExtension(pb extendableProto, extension *ExtensionDesc, value interface{}) error { - if err := checkExtensionTypes(pb, extension); err != nil { +func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error { + epb, ok := extendable(pb) + if !ok { + return errors.New("proto: not an extendable proto") + } + if err := checkExtensionTypes(epb, extension); err != nil { return err } typ := reflect.TypeOf(extension.ExtensionType) @@ -368,10 +543,23 @@ func SetExtension(pb extendableProto, extension *ExtensionDesc, value interface{ return fmt.Errorf("proto: SetExtension called with nil value of type %T", value) } - pb.ExtensionMap()[extension.Field] = Extension{desc: extension, value: value} + extmap := epb.extensionsWrite() + extmap[extension.Field] = Extension{desc: extension, value: value} return nil } +// ClearAllExtensions clears all extensions from pb. +func ClearAllExtensions(pb Message) { + epb, ok := extendable(pb) + if !ok { + return + } + m := epb.extensionsWrite() + for k := range m { + delete(m, k) + } +} + // A global registry of extensions. // The generated code will register the generated descriptors by calling RegisterExtension. diff --git a/vendor/github.com/golang/protobuf/proto/extensions_test.go b/vendor/github.com/golang/protobuf/proto/extensions_test.go deleted file mode 100644 index 80122108..00000000 --- a/vendor/github.com/golang/protobuf/proto/extensions_test.go +++ /dev/null @@ -1,430 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2014 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "bytes" - "fmt" - "reflect" - "testing" - - "github.com/golang/protobuf/proto" - pb "github.com/golang/protobuf/proto/testdata" -) - -func TestGetExtensionsWithMissingExtensions(t *testing.T) { - msg := &pb.MyMessage{} - ext1 := &pb.Ext{} - if err := proto.SetExtension(msg, pb.E_Ext_More, ext1); err != nil { - t.Fatalf("Could not set ext1: %s", ext1) - } - exts, err := proto.GetExtensions(msg, []*proto.ExtensionDesc{ - pb.E_Ext_More, - pb.E_Ext_Text, - }) - if err != nil { - t.Fatalf("GetExtensions() failed: %s", err) - } - if exts[0] != ext1 { - t.Errorf("ext1 not in returned extensions: %T %v", exts[0], exts[0]) - } - if exts[1] != nil { - t.Errorf("ext2 in returned extensions: %T %v", exts[1], exts[1]) - } -} - -func TestGetExtensionStability(t *testing.T) { - check := func(m *pb.MyMessage) bool { - ext1, err := proto.GetExtension(m, pb.E_Ext_More) - if err != nil { - t.Fatalf("GetExtension() failed: %s", err) - } - ext2, err := proto.GetExtension(m, pb.E_Ext_More) - if err != nil { - t.Fatalf("GetExtension() failed: %s", err) - } - return ext1 == ext2 - } - msg := &pb.MyMessage{Count: proto.Int32(4)} - ext0 := &pb.Ext{} - if err := proto.SetExtension(msg, pb.E_Ext_More, ext0); err != nil { - t.Fatalf("Could not set ext1: %s", ext0) - } - if !check(msg) { - t.Errorf("GetExtension() not stable before marshaling") - } - bb, err := proto.Marshal(msg) - if err != nil { - t.Fatalf("Marshal() failed: %s", err) - } - msg1 := &pb.MyMessage{} - err = proto.Unmarshal(bb, msg1) - if err != nil { - t.Fatalf("Unmarshal() failed: %s", err) - } - if !check(msg1) { - t.Errorf("GetExtension() not stable after unmarshaling") - } -} - -func TestGetExtensionDefaults(t *testing.T) { - var setFloat64 float64 = 1 - var setFloat32 float32 = 2 - var setInt32 int32 = 3 - var setInt64 int64 = 4 - var setUint32 uint32 = 5 - var setUint64 uint64 = 6 - var setBool = true - var setBool2 = false - var setString = "Goodnight string" - var setBytes = []byte("Goodnight bytes") - var setEnum = pb.DefaultsMessage_TWO - - type testcase struct { - ext *proto.ExtensionDesc // Extension we are testing. - want interface{} // Expected value of extension, or nil (meaning that GetExtension will fail). - def interface{} // Expected value of extension after ClearExtension(). - } - tests := []testcase{ - {pb.E_NoDefaultDouble, setFloat64, nil}, - {pb.E_NoDefaultFloat, setFloat32, nil}, - {pb.E_NoDefaultInt32, setInt32, nil}, - {pb.E_NoDefaultInt64, setInt64, nil}, - {pb.E_NoDefaultUint32, setUint32, nil}, - {pb.E_NoDefaultUint64, setUint64, nil}, - {pb.E_NoDefaultSint32, setInt32, nil}, - {pb.E_NoDefaultSint64, setInt64, nil}, - {pb.E_NoDefaultFixed32, setUint32, nil}, - {pb.E_NoDefaultFixed64, setUint64, nil}, - {pb.E_NoDefaultSfixed32, setInt32, nil}, - {pb.E_NoDefaultSfixed64, setInt64, nil}, - {pb.E_NoDefaultBool, setBool, nil}, - {pb.E_NoDefaultBool, setBool2, nil}, - {pb.E_NoDefaultString, setString, nil}, - {pb.E_NoDefaultBytes, setBytes, nil}, - {pb.E_NoDefaultEnum, setEnum, nil}, - {pb.E_DefaultDouble, setFloat64, float64(3.1415)}, - {pb.E_DefaultFloat, setFloat32, float32(3.14)}, - {pb.E_DefaultInt32, setInt32, int32(42)}, - {pb.E_DefaultInt64, setInt64, int64(43)}, - {pb.E_DefaultUint32, setUint32, uint32(44)}, - {pb.E_DefaultUint64, setUint64, uint64(45)}, - {pb.E_DefaultSint32, setInt32, int32(46)}, - {pb.E_DefaultSint64, setInt64, int64(47)}, - {pb.E_DefaultFixed32, setUint32, uint32(48)}, - {pb.E_DefaultFixed64, setUint64, uint64(49)}, - {pb.E_DefaultSfixed32, setInt32, int32(50)}, - {pb.E_DefaultSfixed64, setInt64, int64(51)}, - {pb.E_DefaultBool, setBool, true}, - {pb.E_DefaultBool, setBool2, true}, - {pb.E_DefaultString, setString, "Hello, string"}, - {pb.E_DefaultBytes, setBytes, []byte("Hello, bytes")}, - {pb.E_DefaultEnum, setEnum, pb.DefaultsMessage_ONE}, - } - - checkVal := func(test testcase, msg *pb.DefaultsMessage, valWant interface{}) error { - val, err := proto.GetExtension(msg, test.ext) - if err != nil { - if valWant != nil { - return fmt.Errorf("GetExtension(): %s", err) - } - if want := proto.ErrMissingExtension; err != want { - return fmt.Errorf("Unexpected error: got %v, want %v", err, want) - } - return nil - } - - // All proto2 extension values are either a pointer to a value or a slice of values. - ty := reflect.TypeOf(val) - tyWant := reflect.TypeOf(test.ext.ExtensionType) - if got, want := ty, tyWant; got != want { - return fmt.Errorf("unexpected reflect.TypeOf(): got %v want %v", got, want) - } - tye := ty.Elem() - tyeWant := tyWant.Elem() - if got, want := tye, tyeWant; got != want { - return fmt.Errorf("unexpected reflect.TypeOf().Elem(): got %v want %v", got, want) - } - - // Check the name of the type of the value. - // If it is an enum it will be type int32 with the name of the enum. - if got, want := tye.Name(), tye.Name(); got != want { - return fmt.Errorf("unexpected reflect.TypeOf().Elem().Name(): got %v want %v", got, want) - } - - // Check that value is what we expect. - // If we have a pointer in val, get the value it points to. - valExp := val - if ty.Kind() == reflect.Ptr { - valExp = reflect.ValueOf(val).Elem().Interface() - } - if got, want := valExp, valWant; !reflect.DeepEqual(got, want) { - return fmt.Errorf("unexpected reflect.DeepEqual(): got %v want %v", got, want) - } - - return nil - } - - setTo := func(test testcase) interface{} { - setTo := reflect.ValueOf(test.want) - if typ := reflect.TypeOf(test.ext.ExtensionType); typ.Kind() == reflect.Ptr { - setTo = reflect.New(typ).Elem() - setTo.Set(reflect.New(setTo.Type().Elem())) - setTo.Elem().Set(reflect.ValueOf(test.want)) - } - return setTo.Interface() - } - - for _, test := range tests { - msg := &pb.DefaultsMessage{} - name := test.ext.Name - - // Check the initial value. - if err := checkVal(test, msg, test.def); err != nil { - t.Errorf("%s: %v", name, err) - } - - // Set the per-type value and check value. - name = fmt.Sprintf("%s (set to %T %v)", name, test.want, test.want) - if err := proto.SetExtension(msg, test.ext, setTo(test)); err != nil { - t.Errorf("%s: SetExtension(): %v", name, err) - continue - } - if err := checkVal(test, msg, test.want); err != nil { - t.Errorf("%s: %v", name, err) - continue - } - - // Set and check the value. - name += " (cleared)" - proto.ClearExtension(msg, test.ext) - if err := checkVal(test, msg, test.def); err != nil { - t.Errorf("%s: %v", name, err) - } - } -} - -func TestExtensionsRoundTrip(t *testing.T) { - msg := &pb.MyMessage{} - ext1 := &pb.Ext{ - Data: proto.String("hi"), - } - ext2 := &pb.Ext{ - Data: proto.String("there"), - } - exists := proto.HasExtension(msg, pb.E_Ext_More) - if exists { - t.Error("Extension More present unexpectedly") - } - if err := proto.SetExtension(msg, pb.E_Ext_More, ext1); err != nil { - t.Error(err) - } - if err := proto.SetExtension(msg, pb.E_Ext_More, ext2); err != nil { - t.Error(err) - } - e, err := proto.GetExtension(msg, pb.E_Ext_More) - if err != nil { - t.Error(err) - } - x, ok := e.(*pb.Ext) - if !ok { - t.Errorf("e has type %T, expected testdata.Ext", e) - } else if *x.Data != "there" { - t.Errorf("SetExtension failed to overwrite, got %+v, not 'there'", x) - } - proto.ClearExtension(msg, pb.E_Ext_More) - if _, err = proto.GetExtension(msg, pb.E_Ext_More); err != proto.ErrMissingExtension { - t.Errorf("got %v, expected ErrMissingExtension", e) - } - if _, err := proto.GetExtension(msg, pb.E_X215); err == nil { - t.Error("expected bad extension error, got nil") - } - if err := proto.SetExtension(msg, pb.E_X215, 12); err == nil { - t.Error("expected extension err") - } - if err := proto.SetExtension(msg, pb.E_Ext_More, 12); err == nil { - t.Error("expected some sort of type mismatch error, got nil") - } -} - -func TestNilExtension(t *testing.T) { - msg := &pb.MyMessage{ - Count: proto.Int32(1), - } - if err := proto.SetExtension(msg, pb.E_Ext_Text, proto.String("hello")); err != nil { - t.Fatal(err) - } - if err := proto.SetExtension(msg, pb.E_Ext_More, (*pb.Ext)(nil)); err == nil { - t.Error("expected SetExtension to fail due to a nil extension") - } else if want := "proto: SetExtension called with nil value of type *testdata.Ext"; err.Error() != want { - t.Errorf("expected error %v, got %v", want, err) - } - // Note: if the behavior of Marshal is ever changed to ignore nil extensions, update - // this test to verify that E_Ext_Text is properly propagated through marshal->unmarshal. -} - -func TestMarshalUnmarshalRepeatedExtension(t *testing.T) { - // Add a repeated extension to the result. - tests := []struct { - name string - ext []*pb.ComplexExtension - }{ - { - "two fields", - []*pb.ComplexExtension{ - {First: proto.Int32(7)}, - {Second: proto.Int32(11)}, - }, - }, - { - "repeated field", - []*pb.ComplexExtension{ - {Third: []int32{1000}}, - {Third: []int32{2000}}, - }, - }, - { - "two fields and repeated field", - []*pb.ComplexExtension{ - {Third: []int32{1000}}, - {First: proto.Int32(9)}, - {Second: proto.Int32(21)}, - {Third: []int32{2000}}, - }, - }, - } - for _, test := range tests { - // Marshal message with a repeated extension. - msg1 := new(pb.OtherMessage) - err := proto.SetExtension(msg1, pb.E_RComplex, test.ext) - if err != nil { - t.Fatalf("[%s] Error setting extension: %v", test.name, err) - } - b, err := proto.Marshal(msg1) - if err != nil { - t.Fatalf("[%s] Error marshaling message: %v", test.name, err) - } - - // Unmarshal and read the merged proto. - msg2 := new(pb.OtherMessage) - err = proto.Unmarshal(b, msg2) - if err != nil { - t.Fatalf("[%s] Error unmarshaling message: %v", test.name, err) - } - e, err := proto.GetExtension(msg2, pb.E_RComplex) - if err != nil { - t.Fatalf("[%s] Error getting extension: %v", test.name, err) - } - ext := e.([]*pb.ComplexExtension) - if ext == nil { - t.Fatalf("[%s] Invalid extension", test.name) - } - if !reflect.DeepEqual(ext, test.ext) { - t.Errorf("[%s] Wrong value for ComplexExtension: got: %v want: %v\n", test.name, ext, test.ext) - } - } -} - -func TestUnmarshalRepeatingNonRepeatedExtension(t *testing.T) { - // We may see multiple instances of the same extension in the wire - // format. For example, the proto compiler may encode custom options in - // this way. Here, we verify that we merge the extensions together. - tests := []struct { - name string - ext []*pb.ComplexExtension - }{ - { - "two fields", - []*pb.ComplexExtension{ - {First: proto.Int32(7)}, - {Second: proto.Int32(11)}, - }, - }, - { - "repeated field", - []*pb.ComplexExtension{ - {Third: []int32{1000}}, - {Third: []int32{2000}}, - }, - }, - { - "two fields and repeated field", - []*pb.ComplexExtension{ - {Third: []int32{1000}}, - {First: proto.Int32(9)}, - {Second: proto.Int32(21)}, - {Third: []int32{2000}}, - }, - }, - } - for _, test := range tests { - var buf bytes.Buffer - var want pb.ComplexExtension - - // Generate a serialized representation of a repeated extension - // by catenating bytes together. - for i, e := range test.ext { - // Merge to create the wanted proto. - proto.Merge(&want, e) - - // serialize the message - msg := new(pb.OtherMessage) - err := proto.SetExtension(msg, pb.E_Complex, e) - if err != nil { - t.Fatalf("[%s] Error setting extension %d: %v", test.name, i, err) - } - b, err := proto.Marshal(msg) - if err != nil { - t.Fatalf("[%s] Error marshaling message %d: %v", test.name, i, err) - } - buf.Write(b) - } - - // Unmarshal and read the merged proto. - msg2 := new(pb.OtherMessage) - err := proto.Unmarshal(buf.Bytes(), msg2) - if err != nil { - t.Fatalf("[%s] Error unmarshaling message: %v", test.name, err) - } - e, err := proto.GetExtension(msg2, pb.E_Complex) - if err != nil { - t.Fatalf("[%s] Error getting extension: %v", test.name, err) - } - ext := e.(*pb.ComplexExtension) - if ext == nil { - t.Fatalf("[%s] Invalid extension", test.name) - } - if !reflect.DeepEqual(*ext, want) { - t.Errorf("[%s] Wrong value for ComplexExtension: got: %s want: %s\n", test.name, ext, want) - } - } -} diff --git a/vendor/github.com/golang/protobuf/proto/lib.go b/vendor/github.com/golang/protobuf/proto/lib.go index 0de8f8df..ac4ddbc0 100644 --- a/vendor/github.com/golang/protobuf/proto/lib.go +++ b/vendor/github.com/golang/protobuf/proto/lib.go @@ -308,7 +308,7 @@ func GetStats() Stats { return stats } // temporary Buffer and are fine for most applications. type Buffer struct { buf []byte // encode/decode byte stream - index int // write point + index int // read point // pools of basic types to amortize allocation. bools []bool @@ -889,6 +889,10 @@ func isProto3Zero(v reflect.Value) bool { return false } +// ProtoPackageIsVersion2 is referenced from generated protocol buffer files +// to assert that that code is compatible with this version of the proto package. +const ProtoPackageIsVersion2 = true + // ProtoPackageIsVersion1 is referenced from generated protocol buffer files // to assert that that code is compatible with this version of the proto package. const ProtoPackageIsVersion1 = true diff --git a/vendor/github.com/golang/protobuf/proto/message_set.go b/vendor/github.com/golang/protobuf/proto/message_set.go index e25e01e6..fd982dec 100644 --- a/vendor/github.com/golang/protobuf/proto/message_set.go +++ b/vendor/github.com/golang/protobuf/proto/message_set.go @@ -149,9 +149,21 @@ func skipVarint(buf []byte) []byte { // MarshalMessageSet encodes the extension map represented by m in the message set wire format. // It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option. -func MarshalMessageSet(m map[int32]Extension) ([]byte, error) { - if err := encodeExtensionMap(m); err != nil { - return nil, err +func MarshalMessageSet(exts interface{}) ([]byte, error) { + var m map[int32]Extension + switch exts := exts.(type) { + case *XXX_InternalExtensions: + if err := encodeExtensions(exts); err != nil { + return nil, err + } + m, _ = exts.extensionsRead() + case map[int32]Extension: + if err := encodeExtensionsMap(exts); err != nil { + return nil, err + } + m = exts + default: + return nil, errors.New("proto: not an extension map") } // Sort extension IDs to provide a deterministic encoding. @@ -178,7 +190,17 @@ func MarshalMessageSet(m map[int32]Extension) ([]byte, error) { // UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format. // It is called by generated Unmarshal methods on protocol buffer messages with the message_set_wire_format option. -func UnmarshalMessageSet(buf []byte, m map[int32]Extension) error { +func UnmarshalMessageSet(buf []byte, exts interface{}) error { + var m map[int32]Extension + switch exts := exts.(type) { + case *XXX_InternalExtensions: + m = exts.extensionsWrite() + case map[int32]Extension: + m = exts + default: + return errors.New("proto: not an extension map") + } + ms := new(messageSet) if err := Unmarshal(buf, ms); err != nil { return err @@ -209,7 +231,16 @@ func UnmarshalMessageSet(buf []byte, m map[int32]Extension) error { // MarshalMessageSetJSON encodes the extension map represented by m in JSON format. // It is called by generated MarshalJSON methods on protocol buffer messages with the message_set_wire_format option. -func MarshalMessageSetJSON(m map[int32]Extension) ([]byte, error) { +func MarshalMessageSetJSON(exts interface{}) ([]byte, error) { + var m map[int32]Extension + switch exts := exts.(type) { + case *XXX_InternalExtensions: + m, _ = exts.extensionsRead() + case map[int32]Extension: + m = exts + default: + return nil, errors.New("proto: not an extension map") + } var b bytes.Buffer b.WriteByte('{') @@ -252,7 +283,7 @@ func MarshalMessageSetJSON(m map[int32]Extension) ([]byte, error) { // UnmarshalMessageSetJSON decodes the extension map encoded in buf in JSON format. // It is called by generated UnmarshalJSON methods on protocol buffer messages with the message_set_wire_format option. -func UnmarshalMessageSetJSON(buf []byte, m map[int32]Extension) error { +func UnmarshalMessageSetJSON(buf []byte, exts interface{}) error { // Common-case fast path. if len(buf) == 0 || bytes.Equal(buf, []byte("{}")) { return nil diff --git a/vendor/github.com/golang/protobuf/proto/message_set_test.go b/vendor/github.com/golang/protobuf/proto/message_set_test.go deleted file mode 100644 index ab8ac2f0..00000000 --- a/vendor/github.com/golang/protobuf/proto/message_set_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2014 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto - -import ( - "bytes" - "testing" -) - -func TestUnmarshalMessageSetWithDuplicate(t *testing.T) { - // Check that a repeated message set entry will be concatenated. - in := &messageSet{ - Item: []*_MessageSet_Item{ - {TypeId: Int32(12345), Message: []byte("hoo")}, - {TypeId: Int32(12345), Message: []byte("hah")}, - }, - } - b, err := Marshal(in) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - t.Logf("Marshaled bytes: %q", b) - - m := make(map[int32]Extension) - if err := UnmarshalMessageSet(b, m); err != nil { - t.Fatalf("UnmarshalMessageSet: %v", err) - } - ext, ok := m[12345] - if !ok { - t.Fatalf("Didn't retrieve extension 12345; map is %v", m) - } - // Skip wire type/field number and length varints. - got := skipVarint(skipVarint(ext.enc)) - if want := []byte("hoohah"); !bytes.Equal(got, want) { - t.Errorf("Combined extension is %q, want %q", got, want) - } -} diff --git a/vendor/github.com/golang/protobuf/proto/pointer_reflect.go b/vendor/github.com/golang/protobuf/proto/pointer_reflect.go index 749919d2..fb512e2e 100644 --- a/vendor/github.com/golang/protobuf/proto/pointer_reflect.go +++ b/vendor/github.com/golang/protobuf/proto/pointer_reflect.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build appengine +// +build appengine js // This file contains an implementation of proto field accesses using package reflect. // It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can @@ -139,6 +139,11 @@ func structPointer_StringSlice(p structPointer, f field) *[]string { return structPointer_ifield(p, f).(*[]string) } +// Extensions returns the address of an extension map field in the struct. +func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { + return structPointer_ifield(p, f).(*XXX_InternalExtensions) +} + // ExtMap returns the address of an extension map field in the struct. func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { return structPointer_ifield(p, f).(*map[int32]Extension) diff --git a/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go b/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go index e9be0fe9..6b5567d4 100644 --- a/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go +++ b/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build !appengine +// +build !appengine,!js // This file contains the implementation of the proto field accesses using package unsafe. @@ -126,6 +126,10 @@ func structPointer_StringSlice(p structPointer, f field) *[]string { } // ExtMap returns the address of an extension map field in the struct. +func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { + return (*XXX_InternalExtensions)(unsafe.Pointer(uintptr(p) + uintptr(f))) +} + func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { return (*map[int32]Extension)(unsafe.Pointer(uintptr(p) + uintptr(f))) } diff --git a/vendor/github.com/golang/protobuf/proto/properties.go b/vendor/github.com/golang/protobuf/proto/properties.go index 4fe2ec22..ec2289c0 100644 --- a/vendor/github.com/golang/protobuf/proto/properties.go +++ b/vendor/github.com/golang/protobuf/proto/properties.go @@ -473,17 +473,13 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock p.dec = (*Buffer).dec_slice_int64 p.packedDec = (*Buffer).dec_slice_packed_int64 case reflect.Uint8: - p.enc = (*Buffer).enc_slice_byte p.dec = (*Buffer).dec_slice_byte - p.size = size_slice_byte - // This is a []byte, which is either a bytes field, - // or the value of a map field. In the latter case, - // we always encode an empty []byte, so we should not - // use the proto3 enc/size funcs. - // f == nil iff this is the key/value of a map field. - if p.proto3 && f != nil { + if p.proto3 { p.enc = (*Buffer).enc_proto3_slice_byte p.size = size_proto3_slice_byte + } else { + p.enc = (*Buffer).enc_slice_byte + p.size = size_slice_byte } case reflect.Float32, reflect.Float64: switch t2.Bits() { @@ -682,7 +678,8 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { propertiesMap[t] = prop // build properties - prop.extendable = reflect.PtrTo(t).Implements(extendableProtoType) + prop.extendable = reflect.PtrTo(t).Implements(extendableProtoType) || + reflect.PtrTo(t).Implements(extendableProtoV1Type) prop.unrecField = invalidField prop.Prop = make([]*Properties, t.NumField()) prop.order = make([]int, t.NumField()) @@ -693,15 +690,22 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { name := f.Name p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) - if f.Name == "XXX_extensions" { // special case + if f.Name == "XXX_InternalExtensions" { // special case + p.enc = (*Buffer).enc_exts + p.dec = nil // not needed + p.size = size_exts + } else if f.Name == "XXX_extensions" { // special case p.enc = (*Buffer).enc_map p.dec = nil // not needed p.size = size_map - } - if f.Name == "XXX_unrecognized" { // special case + } else if f.Name == "XXX_unrecognized" { // special case prop.unrecField = toField(&f) } - oneof := f.Tag.Get("protobuf_oneof") != "" // special case + oneof := f.Tag.Get("protobuf_oneof") // special case + if oneof != "" { + // Oneof fields don't use the traditional protobuf tag. + p.OrigName = oneof + } prop.Prop[i] = p prop.order[i] = i if debug { @@ -711,7 +715,7 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { } print("\n") } - if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") && !oneof { + if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") && oneof == "" { fmt.Fprintln(os.Stderr, "proto: no encoder for", f.Name, f.Type.String(), "[GetProperties]") } } @@ -840,7 +844,29 @@ func RegisterType(x Message, name string) { } // MessageName returns the fully-qualified proto name for the given message type. -func MessageName(x Message) string { return revProtoTypes[reflect.TypeOf(x)] } +func MessageName(x Message) string { + type xname interface { + XXX_MessageName() string + } + if m, ok := x.(xname); ok { + return m.XXX_MessageName() + } + return revProtoTypes[reflect.TypeOf(x)] +} // MessageType returns the message type (pointer to struct) for a named message. func MessageType(name string) reflect.Type { return protoTypes[name] } + +// A registry of all linked proto files. +var ( + protoFiles = make(map[string][]byte) // file name => fileDescriptor +) + +// RegisterFile is called from generated code and maps from the +// full file name of a .proto file to its compressed FileDescriptorProto. +func RegisterFile(filename string, fileDescriptor []byte) { + protoFiles[filename] = fileDescriptor +} + +// FileDescriptor returns the compressed FileDescriptorProto for a .proto file. +func FileDescriptor(filename string) []byte { return protoFiles[filename] } diff --git a/vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.pb.go b/vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.pb.go deleted file mode 100644 index 37c77820..00000000 --- a/vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.pb.go +++ /dev/null @@ -1,122 +0,0 @@ -// Code generated by protoc-gen-go. -// source: proto3_proto/proto3.proto -// DO NOT EDIT! - -/* -Package proto3_proto is a generated protocol buffer package. - -It is generated from these files: - proto3_proto/proto3.proto - -It has these top-level messages: - Message - Nested - MessageWithMap -*/ -package proto3_proto - -import proto "github.com/golang/protobuf/proto" -import testdata "github.com/golang/protobuf/proto/testdata" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal - -type Message_Humour int32 - -const ( - Message_UNKNOWN Message_Humour = 0 - Message_PUNS Message_Humour = 1 - Message_SLAPSTICK Message_Humour = 2 - Message_BILL_BAILEY Message_Humour = 3 -) - -var Message_Humour_name = map[int32]string{ - 0: "UNKNOWN", - 1: "PUNS", - 2: "SLAPSTICK", - 3: "BILL_BAILEY", -} -var Message_Humour_value = map[string]int32{ - "UNKNOWN": 0, - "PUNS": 1, - "SLAPSTICK": 2, - "BILL_BAILEY": 3, -} - -func (x Message_Humour) String() string { - return proto.EnumName(Message_Humour_name, int32(x)) -} - -type Message struct { - Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - Hilarity Message_Humour `protobuf:"varint,2,opt,name=hilarity,enum=proto3_proto.Message_Humour" json:"hilarity,omitempty"` - HeightInCm uint32 `protobuf:"varint,3,opt,name=height_in_cm" json:"height_in_cm,omitempty"` - Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` - ResultCount int64 `protobuf:"varint,7,opt,name=result_count" json:"result_count,omitempty"` - TrueScotsman bool `protobuf:"varint,8,opt,name=true_scotsman" json:"true_scotsman,omitempty"` - Score float32 `protobuf:"fixed32,9,opt,name=score" json:"score,omitempty"` - Key []uint64 `protobuf:"varint,5,rep,name=key" json:"key,omitempty"` - Nested *Nested `protobuf:"bytes,6,opt,name=nested" json:"nested,omitempty"` - Terrain map[string]*Nested `protobuf:"bytes,10,rep,name=terrain" json:"terrain,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - Proto2Field *testdata.SubDefaults `protobuf:"bytes,11,opt,name=proto2_field" json:"proto2_field,omitempty"` - Proto2Value map[string]*testdata.SubDefaults `protobuf:"bytes,13,rep,name=proto2_value" json:"proto2_value,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` -} - -func (m *Message) Reset() { *m = Message{} } -func (m *Message) String() string { return proto.CompactTextString(m) } -func (*Message) ProtoMessage() {} - -func (m *Message) GetNested() *Nested { - if m != nil { - return m.Nested - } - return nil -} - -func (m *Message) GetTerrain() map[string]*Nested { - if m != nil { - return m.Terrain - } - return nil -} - -func (m *Message) GetProto2Field() *testdata.SubDefaults { - if m != nil { - return m.Proto2Field - } - return nil -} - -func (m *Message) GetProto2Value() map[string]*testdata.SubDefaults { - if m != nil { - return m.Proto2Value - } - return nil -} - -type Nested struct { - Bunny string `protobuf:"bytes,1,opt,name=bunny" json:"bunny,omitempty"` -} - -func (m *Nested) Reset() { *m = Nested{} } -func (m *Nested) String() string { return proto.CompactTextString(m) } -func (*Nested) ProtoMessage() {} - -type MessageWithMap struct { - ByteMapping map[bool][]byte `protobuf:"bytes,1,rep,name=byte_mapping" json:"byte_mapping,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"` -} - -func (m *MessageWithMap) Reset() { *m = MessageWithMap{} } -func (m *MessageWithMap) String() string { return proto.CompactTextString(m) } -func (*MessageWithMap) ProtoMessage() {} - -func (m *MessageWithMap) GetByteMapping() map[bool][]byte { - if m != nil { - return m.ByteMapping - } - return nil -} - -func init() { - proto.RegisterEnum("proto3_proto.Message_Humour", Message_Humour_name, Message_Humour_value) -} diff --git a/vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.proto b/vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.proto deleted file mode 100644 index e2311d92..00000000 --- a/vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.proto +++ /dev/null @@ -1,68 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2014 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto3"; - -import "testdata/test.proto"; - -package proto3_proto; - -message Message { - enum Humour { - UNKNOWN = 0; - PUNS = 1; - SLAPSTICK = 2; - BILL_BAILEY = 3; - } - - string name = 1; - Humour hilarity = 2; - uint32 height_in_cm = 3; - bytes data = 4; - int64 result_count = 7; - bool true_scotsman = 8; - float score = 9; - - repeated uint64 key = 5; - Nested nested = 6; - - map terrain = 10; - testdata.SubDefaults proto2_field = 11; - map proto2_value = 13; -} - -message Nested { - string bunny = 1; -} - -message MessageWithMap { - map byte_mapping = 1; -} diff --git a/vendor/github.com/golang/protobuf/proto/proto3_test.go b/vendor/github.com/golang/protobuf/proto/proto3_test.go deleted file mode 100644 index 462f8055..00000000 --- a/vendor/github.com/golang/protobuf/proto/proto3_test.go +++ /dev/null @@ -1,125 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2014 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "testing" - - "github.com/golang/protobuf/proto" - pb "github.com/golang/protobuf/proto/proto3_proto" - tpb "github.com/golang/protobuf/proto/testdata" -) - -func TestProto3ZeroValues(t *testing.T) { - tests := []struct { - desc string - m proto.Message - }{ - {"zero message", &pb.Message{}}, - {"empty bytes field", &pb.Message{Data: []byte{}}}, - } - for _, test := range tests { - b, err := proto.Marshal(test.m) - if err != nil { - t.Errorf("%s: proto.Marshal: %v", test.desc, err) - continue - } - if len(b) > 0 { - t.Errorf("%s: Encoding is non-empty: %q", test.desc, b) - } - } -} - -func TestRoundTripProto3(t *testing.T) { - m := &pb.Message{ - Name: "David", // (2 | 1<<3): 0x0a 0x05 "David" - Hilarity: pb.Message_PUNS, // (0 | 2<<3): 0x10 0x01 - HeightInCm: 178, // (0 | 3<<3): 0x18 0xb2 0x01 - Data: []byte("roboto"), // (2 | 4<<3): 0x20 0x06 "roboto" - ResultCount: 47, // (0 | 7<<3): 0x38 0x2f - TrueScotsman: true, // (0 | 8<<3): 0x40 0x01 - Score: 8.1, // (5 | 9<<3): 0x4d <8.1> - - Key: []uint64{1, 0xdeadbeef}, - Nested: &pb.Nested{ - Bunny: "Monty", - }, - } - t.Logf(" m: %v", m) - - b, err := proto.Marshal(m) - if err != nil { - t.Fatalf("proto.Marshal: %v", err) - } - t.Logf(" b: %q", b) - - m2 := new(pb.Message) - if err := proto.Unmarshal(b, m2); err != nil { - t.Fatalf("proto.Unmarshal: %v", err) - } - t.Logf("m2: %v", m2) - - if !proto.Equal(m, m2) { - t.Errorf("proto.Equal returned false:\n m: %v\nm2: %v", m, m2) - } -} - -func TestProto3SetDefaults(t *testing.T) { - in := &pb.Message{ - Terrain: map[string]*pb.Nested{ - "meadow": new(pb.Nested), - }, - Proto2Field: new(tpb.SubDefaults), - Proto2Value: map[string]*tpb.SubDefaults{ - "badlands": new(tpb.SubDefaults), - }, - } - - got := proto.Clone(in).(*pb.Message) - proto.SetDefaults(got) - - // There are no defaults in proto3. Everything should be the zero value, but - // we need to remember to set defaults for nested proto2 messages. - want := &pb.Message{ - Terrain: map[string]*pb.Nested{ - "meadow": new(pb.Nested), - }, - Proto2Field: &tpb.SubDefaults{N: proto.Int64(7)}, - Proto2Value: map[string]*tpb.SubDefaults{ - "badlands": &tpb.SubDefaults{N: proto.Int64(7)}, - }, - } - - if !proto.Equal(got, want) { - t.Errorf("with in = %v\nproto.SetDefaults(in) =>\ngot %v\nwant %v", in, got, want) - } -} diff --git a/vendor/github.com/golang/protobuf/proto/size2_test.go b/vendor/github.com/golang/protobuf/proto/size2_test.go deleted file mode 100644 index a2729c39..00000000 --- a/vendor/github.com/golang/protobuf/proto/size2_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2012 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto - -import ( - "testing" -) - -// This is a separate file and package from size_test.go because that one uses -// generated messages and thus may not be in package proto without having a circular -// dependency, whereas this file tests unexported details of size.go. - -func TestVarintSize(t *testing.T) { - // Check the edge cases carefully. - testCases := []struct { - n uint64 - size int - }{ - {0, 1}, - {1, 1}, - {127, 1}, - {128, 2}, - {16383, 2}, - {16384, 3}, - {1<<63 - 1, 9}, - {1 << 63, 10}, - } - for _, tc := range testCases { - size := sizeVarint(tc.n) - if size != tc.size { - t.Errorf("sizeVarint(%d) = %d, want %d", tc.n, size, tc.size) - } - } -} diff --git a/vendor/github.com/golang/protobuf/proto/size_test.go b/vendor/github.com/golang/protobuf/proto/size_test.go deleted file mode 100644 index af1034dc..00000000 --- a/vendor/github.com/golang/protobuf/proto/size_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2012 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "log" - "strings" - "testing" - - . "github.com/golang/protobuf/proto" - proto3pb "github.com/golang/protobuf/proto/proto3_proto" - pb "github.com/golang/protobuf/proto/testdata" -) - -var messageWithExtension1 = &pb.MyMessage{Count: Int32(7)} - -// messageWithExtension2 is in equal_test.go. -var messageWithExtension3 = &pb.MyMessage{Count: Int32(8)} - -func init() { - if err := SetExtension(messageWithExtension1, pb.E_Ext_More, &pb.Ext{Data: String("Abbott")}); err != nil { - log.Panicf("SetExtension: %v", err) - } - if err := SetExtension(messageWithExtension3, pb.E_Ext_More, &pb.Ext{Data: String("Costello")}); err != nil { - log.Panicf("SetExtension: %v", err) - } - - // Force messageWithExtension3 to have the extension encoded. - Marshal(messageWithExtension3) - -} - -var SizeTests = []struct { - desc string - pb Message -}{ - {"empty", &pb.OtherMessage{}}, - // Basic types. - {"bool", &pb.Defaults{F_Bool: Bool(true)}}, - {"int32", &pb.Defaults{F_Int32: Int32(12)}}, - {"negative int32", &pb.Defaults{F_Int32: Int32(-1)}}, - {"small int64", &pb.Defaults{F_Int64: Int64(1)}}, - {"big int64", &pb.Defaults{F_Int64: Int64(1 << 20)}}, - {"negative int64", &pb.Defaults{F_Int64: Int64(-1)}}, - {"fixed32", &pb.Defaults{F_Fixed32: Uint32(71)}}, - {"fixed64", &pb.Defaults{F_Fixed64: Uint64(72)}}, - {"uint32", &pb.Defaults{F_Uint32: Uint32(123)}}, - {"uint64", &pb.Defaults{F_Uint64: Uint64(124)}}, - {"float", &pb.Defaults{F_Float: Float32(12.6)}}, - {"double", &pb.Defaults{F_Double: Float64(13.9)}}, - {"string", &pb.Defaults{F_String: String("niles")}}, - {"bytes", &pb.Defaults{F_Bytes: []byte("wowsa")}}, - {"bytes, empty", &pb.Defaults{F_Bytes: []byte{}}}, - {"sint32", &pb.Defaults{F_Sint32: Int32(65)}}, - {"sint64", &pb.Defaults{F_Sint64: Int64(67)}}, - {"enum", &pb.Defaults{F_Enum: pb.Defaults_BLUE.Enum()}}, - // Repeated. - {"empty repeated bool", &pb.MoreRepeated{Bools: []bool{}}}, - {"repeated bool", &pb.MoreRepeated{Bools: []bool{false, true, true, false}}}, - {"packed repeated bool", &pb.MoreRepeated{BoolsPacked: []bool{false, true, true, false, true, true, true}}}, - {"repeated int32", &pb.MoreRepeated{Ints: []int32{1, 12203, 1729, -1}}}, - {"repeated int32 packed", &pb.MoreRepeated{IntsPacked: []int32{1, 12203, 1729}}}, - {"repeated int64 packed", &pb.MoreRepeated{Int64SPacked: []int64{ - // Need enough large numbers to verify that the header is counting the number of bytes - // for the field, not the number of elements. - 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, - 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, 1 << 62, - }}}, - {"repeated string", &pb.MoreRepeated{Strings: []string{"r", "ken", "gri"}}}, - {"repeated fixed", &pb.MoreRepeated{Fixeds: []uint32{1, 2, 3, 4}}}, - // Nested. - {"nested", &pb.OldMessage{Nested: &pb.OldMessage_Nested{Name: String("whatever")}}}, - {"group", &pb.GroupOld{G: &pb.GroupOld_G{X: Int32(12345)}}}, - // Other things. - {"unrecognized", &pb.MoreRepeated{XXX_unrecognized: []byte{13<<3 | 0, 4}}}, - {"extension (unencoded)", messageWithExtension1}, - {"extension (encoded)", messageWithExtension3}, - // proto3 message - {"proto3 empty", &proto3pb.Message{}}, - {"proto3 bool", &proto3pb.Message{TrueScotsman: true}}, - {"proto3 int64", &proto3pb.Message{ResultCount: 1}}, - {"proto3 uint32", &proto3pb.Message{HeightInCm: 123}}, - {"proto3 float", &proto3pb.Message{Score: 12.6}}, - {"proto3 string", &proto3pb.Message{Name: "Snezana"}}, - {"proto3 bytes", &proto3pb.Message{Data: []byte("wowsa")}}, - {"proto3 bytes, empty", &proto3pb.Message{Data: []byte{}}}, - {"proto3 enum", &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, - {"proto3 map field with empty bytes", &proto3pb.MessageWithMap{ByteMapping: map[bool][]byte{false: []byte{}}}}, - - {"map field", &pb.MessageWithMap{NameMapping: map[int32]string{1: "Rob", 7: "Andrew"}}}, - {"map field with message", &pb.MessageWithMap{MsgMapping: map[int64]*pb.FloatingPoint{0x7001: &pb.FloatingPoint{F: Float64(2.0)}}}}, - {"map field with bytes", &pb.MessageWithMap{ByteMapping: map[bool][]byte{true: []byte("this time for sure")}}}, - {"map field with empty bytes", &pb.MessageWithMap{ByteMapping: map[bool][]byte{true: []byte{}}}}, - - {"map field with big entry", &pb.MessageWithMap{NameMapping: map[int32]string{8: strings.Repeat("x", 125)}}}, - {"map field with big key and val", &pb.MessageWithMap{StrToStr: map[string]string{strings.Repeat("x", 70): strings.Repeat("y", 70)}}}, - {"map field with big numeric key", &pb.MessageWithMap{NameMapping: map[int32]string{0xf00d: "om nom nom"}}}, - - {"oneof not set", &pb.Oneof{}}, - {"oneof bool", &pb.Oneof{Union: &pb.Oneof_F_Bool{true}}}, - {"oneof zero int32", &pb.Oneof{Union: &pb.Oneof_F_Int32{0}}}, - {"oneof big int32", &pb.Oneof{Union: &pb.Oneof_F_Int32{1 << 20}}}, - {"oneof int64", &pb.Oneof{Union: &pb.Oneof_F_Int64{42}}}, - {"oneof fixed32", &pb.Oneof{Union: &pb.Oneof_F_Fixed32{43}}}, - {"oneof fixed64", &pb.Oneof{Union: &pb.Oneof_F_Fixed64{44}}}, - {"oneof uint32", &pb.Oneof{Union: &pb.Oneof_F_Uint32{45}}}, - {"oneof uint64", &pb.Oneof{Union: &pb.Oneof_F_Uint64{46}}}, - {"oneof float", &pb.Oneof{Union: &pb.Oneof_F_Float{47.1}}}, - {"oneof double", &pb.Oneof{Union: &pb.Oneof_F_Double{48.9}}}, - {"oneof string", &pb.Oneof{Union: &pb.Oneof_F_String{"Rhythmic Fman"}}}, - {"oneof bytes", &pb.Oneof{Union: &pb.Oneof_F_Bytes{[]byte("let go")}}}, - {"oneof sint32", &pb.Oneof{Union: &pb.Oneof_F_Sint32{50}}}, - {"oneof sint64", &pb.Oneof{Union: &pb.Oneof_F_Sint64{51}}}, - {"oneof enum", &pb.Oneof{Union: &pb.Oneof_F_Enum{pb.MyMessage_BLUE}}}, - {"message for oneof", &pb.GoTestField{Label: String("k"), Type: String("v")}}, - {"oneof message", &pb.Oneof{Union: &pb.Oneof_F_Message{&pb.GoTestField{Label: String("k"), Type: String("v")}}}}, - {"oneof group", &pb.Oneof{Union: &pb.Oneof_FGroup{&pb.Oneof_F_Group{X: Int32(52)}}}}, - {"oneof largest tag", &pb.Oneof{Union: &pb.Oneof_F_Largest_Tag{1}}}, - {"multiple oneofs", &pb.Oneof{Union: &pb.Oneof_F_Int32{1}, Tormato: &pb.Oneof_Value{2}}}, -} - -func TestSize(t *testing.T) { - for _, tc := range SizeTests { - size := Size(tc.pb) - b, err := Marshal(tc.pb) - if err != nil { - t.Errorf("%v: Marshal failed: %v", tc.desc, err) - continue - } - if size != len(b) { - t.Errorf("%v: Size(%v) = %d, want %d", tc.desc, tc.pb, size, len(b)) - t.Logf("%v: bytes: %#v", tc.desc, b) - } - } -} diff --git a/vendor/github.com/golang/protobuf/proto/testdata/Makefile b/vendor/github.com/golang/protobuf/proto/testdata/Makefile deleted file mode 100644 index fc288628..00000000 --- a/vendor/github.com/golang/protobuf/proto/testdata/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# Go support for Protocol Buffers - Google's data interchange format -# -# Copyright 2010 The Go Authors. All rights reserved. -# https://github.com/golang/protobuf -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -include ../../Make.protobuf - -all: regenerate - -regenerate: - rm -f test.pb.go - make test.pb.go - -# The following rules are just aids to development. Not needed for typical testing. - -diff: regenerate - git diff test.pb.go - -restore: - cp test.pb.go.golden test.pb.go - -preserve: - cp test.pb.go test.pb.go.golden diff --git a/vendor/github.com/golang/protobuf/proto/testdata/golden_test.go b/vendor/github.com/golang/protobuf/proto/testdata/golden_test.go deleted file mode 100644 index 7172d0e9..00000000 --- a/vendor/github.com/golang/protobuf/proto/testdata/golden_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2012 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Verify that the compiler output for test.proto is unchanged. - -package testdata - -import ( - "crypto/sha1" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "testing" -) - -// sum returns in string form (for easy comparison) the SHA-1 hash of the named file. -func sum(t *testing.T, name string) string { - data, err := ioutil.ReadFile(name) - if err != nil { - t.Fatal(err) - } - t.Logf("sum(%q): length is %d", name, len(data)) - hash := sha1.New() - _, err = hash.Write(data) - if err != nil { - t.Fatal(err) - } - return fmt.Sprintf("% x", hash.Sum(nil)) -} - -func run(t *testing.T, name string, args ...string) { - cmd := exec.Command(name, args...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err := cmd.Run() - if err != nil { - t.Fatal(err) - } -} - -func TestGolden(t *testing.T) { - // Compute the original checksum. - goldenSum := sum(t, "test.pb.go") - // Run the proto compiler. - run(t, "protoc", "--go_out="+os.TempDir(), "test.proto") - newFile := filepath.Join(os.TempDir(), "test.pb.go") - defer os.Remove(newFile) - // Compute the new checksum. - newSum := sum(t, newFile) - // Verify - if newSum != goldenSum { - run(t, "diff", "-u", "test.pb.go", newFile) - t.Fatal("Code generated by protoc-gen-go has changed; update test.pb.go") - } -} diff --git a/vendor/github.com/golang/protobuf/proto/testdata/test.pb.go b/vendor/github.com/golang/protobuf/proto/testdata/test.pb.go deleted file mode 100644 index b04d480a..00000000 --- a/vendor/github.com/golang/protobuf/proto/testdata/test.pb.go +++ /dev/null @@ -1,3992 +0,0 @@ -// Code generated by protoc-gen-go. -// source: test.proto -// DO NOT EDIT! - -/* -Package testdata is a generated protocol buffer package. - -It is generated from these files: - test.proto - -It has these top-level messages: - GoEnum - GoTestField - GoTest - GoSkipTest - NonPackedTest - PackedTest - MaxTag - OldMessage - NewMessage - InnerMessage - OtherMessage - MyMessage - Ext - ComplexExtension - DefaultsMessage - MyMessageSet - Empty - MessageList - Strings - Defaults - SubDefaults - RepeatedEnum - MoreRepeated - GroupOld - GroupNew - FloatingPoint - MessageWithMap - Oneof - Communique -*/ -package testdata - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -const _ = proto.ProtoPackageIsVersion1 - -type FOO int32 - -const ( - FOO_FOO1 FOO = 1 -) - -var FOO_name = map[int32]string{ - 1: "FOO1", -} -var FOO_value = map[string]int32{ - "FOO1": 1, -} - -func (x FOO) Enum() *FOO { - p := new(FOO) - *p = x - return p -} -func (x FOO) String() string { - return proto.EnumName(FOO_name, int32(x)) -} -func (x *FOO) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(FOO_value, data, "FOO") - if err != nil { - return err - } - *x = FOO(value) - return nil -} -func (FOO) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } - -// An enum, for completeness. -type GoTest_KIND int32 - -const ( - GoTest_VOID GoTest_KIND = 0 - // Basic types - GoTest_BOOL GoTest_KIND = 1 - GoTest_BYTES GoTest_KIND = 2 - GoTest_FINGERPRINT GoTest_KIND = 3 - GoTest_FLOAT GoTest_KIND = 4 - GoTest_INT GoTest_KIND = 5 - GoTest_STRING GoTest_KIND = 6 - GoTest_TIME GoTest_KIND = 7 - // Groupings - GoTest_TUPLE GoTest_KIND = 8 - GoTest_ARRAY GoTest_KIND = 9 - GoTest_MAP GoTest_KIND = 10 - // Table types - GoTest_TABLE GoTest_KIND = 11 - // Functions - GoTest_FUNCTION GoTest_KIND = 12 -) - -var GoTest_KIND_name = map[int32]string{ - 0: "VOID", - 1: "BOOL", - 2: "BYTES", - 3: "FINGERPRINT", - 4: "FLOAT", - 5: "INT", - 6: "STRING", - 7: "TIME", - 8: "TUPLE", - 9: "ARRAY", - 10: "MAP", - 11: "TABLE", - 12: "FUNCTION", -} -var GoTest_KIND_value = map[string]int32{ - "VOID": 0, - "BOOL": 1, - "BYTES": 2, - "FINGERPRINT": 3, - "FLOAT": 4, - "INT": 5, - "STRING": 6, - "TIME": 7, - "TUPLE": 8, - "ARRAY": 9, - "MAP": 10, - "TABLE": 11, - "FUNCTION": 12, -} - -func (x GoTest_KIND) Enum() *GoTest_KIND { - p := new(GoTest_KIND) - *p = x - return p -} -func (x GoTest_KIND) String() string { - return proto.EnumName(GoTest_KIND_name, int32(x)) -} -func (x *GoTest_KIND) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(GoTest_KIND_value, data, "GoTest_KIND") - if err != nil { - return err - } - *x = GoTest_KIND(value) - return nil -} -func (GoTest_KIND) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{2, 0} } - -type MyMessage_Color int32 - -const ( - MyMessage_RED MyMessage_Color = 0 - MyMessage_GREEN MyMessage_Color = 1 - MyMessage_BLUE MyMessage_Color = 2 -) - -var MyMessage_Color_name = map[int32]string{ - 0: "RED", - 1: "GREEN", - 2: "BLUE", -} -var MyMessage_Color_value = map[string]int32{ - "RED": 0, - "GREEN": 1, - "BLUE": 2, -} - -func (x MyMessage_Color) Enum() *MyMessage_Color { - p := new(MyMessage_Color) - *p = x - return p -} -func (x MyMessage_Color) String() string { - return proto.EnumName(MyMessage_Color_name, int32(x)) -} -func (x *MyMessage_Color) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(MyMessage_Color_value, data, "MyMessage_Color") - if err != nil { - return err - } - *x = MyMessage_Color(value) - return nil -} -func (MyMessage_Color) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{11, 0} } - -type DefaultsMessage_DefaultsEnum int32 - -const ( - DefaultsMessage_ZERO DefaultsMessage_DefaultsEnum = 0 - DefaultsMessage_ONE DefaultsMessage_DefaultsEnum = 1 - DefaultsMessage_TWO DefaultsMessage_DefaultsEnum = 2 -) - -var DefaultsMessage_DefaultsEnum_name = map[int32]string{ - 0: "ZERO", - 1: "ONE", - 2: "TWO", -} -var DefaultsMessage_DefaultsEnum_value = map[string]int32{ - "ZERO": 0, - "ONE": 1, - "TWO": 2, -} - -func (x DefaultsMessage_DefaultsEnum) Enum() *DefaultsMessage_DefaultsEnum { - p := new(DefaultsMessage_DefaultsEnum) - *p = x - return p -} -func (x DefaultsMessage_DefaultsEnum) String() string { - return proto.EnumName(DefaultsMessage_DefaultsEnum_name, int32(x)) -} -func (x *DefaultsMessage_DefaultsEnum) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(DefaultsMessage_DefaultsEnum_value, data, "DefaultsMessage_DefaultsEnum") - if err != nil { - return err - } - *x = DefaultsMessage_DefaultsEnum(value) - return nil -} -func (DefaultsMessage_DefaultsEnum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor0, []int{14, 0} -} - -type Defaults_Color int32 - -const ( - Defaults_RED Defaults_Color = 0 - Defaults_GREEN Defaults_Color = 1 - Defaults_BLUE Defaults_Color = 2 -) - -var Defaults_Color_name = map[int32]string{ - 0: "RED", - 1: "GREEN", - 2: "BLUE", -} -var Defaults_Color_value = map[string]int32{ - "RED": 0, - "GREEN": 1, - "BLUE": 2, -} - -func (x Defaults_Color) Enum() *Defaults_Color { - p := new(Defaults_Color) - *p = x - return p -} -func (x Defaults_Color) String() string { - return proto.EnumName(Defaults_Color_name, int32(x)) -} -func (x *Defaults_Color) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(Defaults_Color_value, data, "Defaults_Color") - if err != nil { - return err - } - *x = Defaults_Color(value) - return nil -} -func (Defaults_Color) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{19, 0} } - -type RepeatedEnum_Color int32 - -const ( - RepeatedEnum_RED RepeatedEnum_Color = 1 -) - -var RepeatedEnum_Color_name = map[int32]string{ - 1: "RED", -} -var RepeatedEnum_Color_value = map[string]int32{ - "RED": 1, -} - -func (x RepeatedEnum_Color) Enum() *RepeatedEnum_Color { - p := new(RepeatedEnum_Color) - *p = x - return p -} -func (x RepeatedEnum_Color) String() string { - return proto.EnumName(RepeatedEnum_Color_name, int32(x)) -} -func (x *RepeatedEnum_Color) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(RepeatedEnum_Color_value, data, "RepeatedEnum_Color") - if err != nil { - return err - } - *x = RepeatedEnum_Color(value) - return nil -} -func (RepeatedEnum_Color) EnumDescriptor() ([]byte, []int) { return fileDescriptor0, []int{21, 0} } - -type GoEnum struct { - Foo *FOO `protobuf:"varint,1,req,name=foo,enum=testdata.FOO" json:"foo,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoEnum) Reset() { *m = GoEnum{} } -func (m *GoEnum) String() string { return proto.CompactTextString(m) } -func (*GoEnum) ProtoMessage() {} -func (*GoEnum) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } - -func (m *GoEnum) GetFoo() FOO { - if m != nil && m.Foo != nil { - return *m.Foo - } - return FOO_FOO1 -} - -type GoTestField struct { - Label *string `protobuf:"bytes,1,req,name=Label,json=label" json:"Label,omitempty"` - Type *string `protobuf:"bytes,2,req,name=Type,json=type" json:"Type,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoTestField) Reset() { *m = GoTestField{} } -func (m *GoTestField) String() string { return proto.CompactTextString(m) } -func (*GoTestField) ProtoMessage() {} -func (*GoTestField) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } - -func (m *GoTestField) GetLabel() string { - if m != nil && m.Label != nil { - return *m.Label - } - return "" -} - -func (m *GoTestField) GetType() string { - if m != nil && m.Type != nil { - return *m.Type - } - return "" -} - -type GoTest struct { - // Some typical parameters - Kind *GoTest_KIND `protobuf:"varint,1,req,name=Kind,json=kind,enum=testdata.GoTest_KIND" json:"Kind,omitempty"` - Table *string `protobuf:"bytes,2,opt,name=Table,json=table" json:"Table,omitempty"` - Param *int32 `protobuf:"varint,3,opt,name=Param,json=param" json:"Param,omitempty"` - // Required, repeated and optional foreign fields. - RequiredField *GoTestField `protobuf:"bytes,4,req,name=RequiredField,json=requiredField" json:"RequiredField,omitempty"` - RepeatedField []*GoTestField `protobuf:"bytes,5,rep,name=RepeatedField,json=repeatedField" json:"RepeatedField,omitempty"` - OptionalField *GoTestField `protobuf:"bytes,6,opt,name=OptionalField,json=optionalField" json:"OptionalField,omitempty"` - // Required fields of all basic types - F_BoolRequired *bool `protobuf:"varint,10,req,name=F_Bool_required,json=fBoolRequired" json:"F_Bool_required,omitempty"` - F_Int32Required *int32 `protobuf:"varint,11,req,name=F_Int32_required,json=fInt32Required" json:"F_Int32_required,omitempty"` - F_Int64Required *int64 `protobuf:"varint,12,req,name=F_Int64_required,json=fInt64Required" json:"F_Int64_required,omitempty"` - F_Fixed32Required *uint32 `protobuf:"fixed32,13,req,name=F_Fixed32_required,json=fFixed32Required" json:"F_Fixed32_required,omitempty"` - F_Fixed64Required *uint64 `protobuf:"fixed64,14,req,name=F_Fixed64_required,json=fFixed64Required" json:"F_Fixed64_required,omitempty"` - F_Uint32Required *uint32 `protobuf:"varint,15,req,name=F_Uint32_required,json=fUint32Required" json:"F_Uint32_required,omitempty"` - F_Uint64Required *uint64 `protobuf:"varint,16,req,name=F_Uint64_required,json=fUint64Required" json:"F_Uint64_required,omitempty"` - F_FloatRequired *float32 `protobuf:"fixed32,17,req,name=F_Float_required,json=fFloatRequired" json:"F_Float_required,omitempty"` - F_DoubleRequired *float64 `protobuf:"fixed64,18,req,name=F_Double_required,json=fDoubleRequired" json:"F_Double_required,omitempty"` - F_StringRequired *string `protobuf:"bytes,19,req,name=F_String_required,json=fStringRequired" json:"F_String_required,omitempty"` - F_BytesRequired []byte `protobuf:"bytes,101,req,name=F_Bytes_required,json=fBytesRequired" json:"F_Bytes_required,omitempty"` - F_Sint32Required *int32 `protobuf:"zigzag32,102,req,name=F_Sint32_required,json=fSint32Required" json:"F_Sint32_required,omitempty"` - F_Sint64Required *int64 `protobuf:"zigzag64,103,req,name=F_Sint64_required,json=fSint64Required" json:"F_Sint64_required,omitempty"` - // Repeated fields of all basic types - F_BoolRepeated []bool `protobuf:"varint,20,rep,name=F_Bool_repeated,json=fBoolRepeated" json:"F_Bool_repeated,omitempty"` - F_Int32Repeated []int32 `protobuf:"varint,21,rep,name=F_Int32_repeated,json=fInt32Repeated" json:"F_Int32_repeated,omitempty"` - F_Int64Repeated []int64 `protobuf:"varint,22,rep,name=F_Int64_repeated,json=fInt64Repeated" json:"F_Int64_repeated,omitempty"` - F_Fixed32Repeated []uint32 `protobuf:"fixed32,23,rep,name=F_Fixed32_repeated,json=fFixed32Repeated" json:"F_Fixed32_repeated,omitempty"` - F_Fixed64Repeated []uint64 `protobuf:"fixed64,24,rep,name=F_Fixed64_repeated,json=fFixed64Repeated" json:"F_Fixed64_repeated,omitempty"` - F_Uint32Repeated []uint32 `protobuf:"varint,25,rep,name=F_Uint32_repeated,json=fUint32Repeated" json:"F_Uint32_repeated,omitempty"` - F_Uint64Repeated []uint64 `protobuf:"varint,26,rep,name=F_Uint64_repeated,json=fUint64Repeated" json:"F_Uint64_repeated,omitempty"` - F_FloatRepeated []float32 `protobuf:"fixed32,27,rep,name=F_Float_repeated,json=fFloatRepeated" json:"F_Float_repeated,omitempty"` - F_DoubleRepeated []float64 `protobuf:"fixed64,28,rep,name=F_Double_repeated,json=fDoubleRepeated" json:"F_Double_repeated,omitempty"` - F_StringRepeated []string `protobuf:"bytes,29,rep,name=F_String_repeated,json=fStringRepeated" json:"F_String_repeated,omitempty"` - F_BytesRepeated [][]byte `protobuf:"bytes,201,rep,name=F_Bytes_repeated,json=fBytesRepeated" json:"F_Bytes_repeated,omitempty"` - F_Sint32Repeated []int32 `protobuf:"zigzag32,202,rep,name=F_Sint32_repeated,json=fSint32Repeated" json:"F_Sint32_repeated,omitempty"` - F_Sint64Repeated []int64 `protobuf:"zigzag64,203,rep,name=F_Sint64_repeated,json=fSint64Repeated" json:"F_Sint64_repeated,omitempty"` - // Optional fields of all basic types - F_BoolOptional *bool `protobuf:"varint,30,opt,name=F_Bool_optional,json=fBoolOptional" json:"F_Bool_optional,omitempty"` - F_Int32Optional *int32 `protobuf:"varint,31,opt,name=F_Int32_optional,json=fInt32Optional" json:"F_Int32_optional,omitempty"` - F_Int64Optional *int64 `protobuf:"varint,32,opt,name=F_Int64_optional,json=fInt64Optional" json:"F_Int64_optional,omitempty"` - F_Fixed32Optional *uint32 `protobuf:"fixed32,33,opt,name=F_Fixed32_optional,json=fFixed32Optional" json:"F_Fixed32_optional,omitempty"` - F_Fixed64Optional *uint64 `protobuf:"fixed64,34,opt,name=F_Fixed64_optional,json=fFixed64Optional" json:"F_Fixed64_optional,omitempty"` - F_Uint32Optional *uint32 `protobuf:"varint,35,opt,name=F_Uint32_optional,json=fUint32Optional" json:"F_Uint32_optional,omitempty"` - F_Uint64Optional *uint64 `protobuf:"varint,36,opt,name=F_Uint64_optional,json=fUint64Optional" json:"F_Uint64_optional,omitempty"` - F_FloatOptional *float32 `protobuf:"fixed32,37,opt,name=F_Float_optional,json=fFloatOptional" json:"F_Float_optional,omitempty"` - F_DoubleOptional *float64 `protobuf:"fixed64,38,opt,name=F_Double_optional,json=fDoubleOptional" json:"F_Double_optional,omitempty"` - F_StringOptional *string `protobuf:"bytes,39,opt,name=F_String_optional,json=fStringOptional" json:"F_String_optional,omitempty"` - F_BytesOptional []byte `protobuf:"bytes,301,opt,name=F_Bytes_optional,json=fBytesOptional" json:"F_Bytes_optional,omitempty"` - F_Sint32Optional *int32 `protobuf:"zigzag32,302,opt,name=F_Sint32_optional,json=fSint32Optional" json:"F_Sint32_optional,omitempty"` - F_Sint64Optional *int64 `protobuf:"zigzag64,303,opt,name=F_Sint64_optional,json=fSint64Optional" json:"F_Sint64_optional,omitempty"` - // Default-valued fields of all basic types - F_BoolDefaulted *bool `protobuf:"varint,40,opt,name=F_Bool_defaulted,json=fBoolDefaulted,def=1" json:"F_Bool_defaulted,omitempty"` - F_Int32Defaulted *int32 `protobuf:"varint,41,opt,name=F_Int32_defaulted,json=fInt32Defaulted,def=32" json:"F_Int32_defaulted,omitempty"` - F_Int64Defaulted *int64 `protobuf:"varint,42,opt,name=F_Int64_defaulted,json=fInt64Defaulted,def=64" json:"F_Int64_defaulted,omitempty"` - F_Fixed32Defaulted *uint32 `protobuf:"fixed32,43,opt,name=F_Fixed32_defaulted,json=fFixed32Defaulted,def=320" json:"F_Fixed32_defaulted,omitempty"` - F_Fixed64Defaulted *uint64 `protobuf:"fixed64,44,opt,name=F_Fixed64_defaulted,json=fFixed64Defaulted,def=640" json:"F_Fixed64_defaulted,omitempty"` - F_Uint32Defaulted *uint32 `protobuf:"varint,45,opt,name=F_Uint32_defaulted,json=fUint32Defaulted,def=3200" json:"F_Uint32_defaulted,omitempty"` - F_Uint64Defaulted *uint64 `protobuf:"varint,46,opt,name=F_Uint64_defaulted,json=fUint64Defaulted,def=6400" json:"F_Uint64_defaulted,omitempty"` - F_FloatDefaulted *float32 `protobuf:"fixed32,47,opt,name=F_Float_defaulted,json=fFloatDefaulted,def=314159" json:"F_Float_defaulted,omitempty"` - F_DoubleDefaulted *float64 `protobuf:"fixed64,48,opt,name=F_Double_defaulted,json=fDoubleDefaulted,def=271828" json:"F_Double_defaulted,omitempty"` - F_StringDefaulted *string `protobuf:"bytes,49,opt,name=F_String_defaulted,json=fStringDefaulted,def=hello, \"world!\"\n" json:"F_String_defaulted,omitempty"` - F_BytesDefaulted []byte `protobuf:"bytes,401,opt,name=F_Bytes_defaulted,json=fBytesDefaulted,def=Bignose" json:"F_Bytes_defaulted,omitempty"` - F_Sint32Defaulted *int32 `protobuf:"zigzag32,402,opt,name=F_Sint32_defaulted,json=fSint32Defaulted,def=-32" json:"F_Sint32_defaulted,omitempty"` - F_Sint64Defaulted *int64 `protobuf:"zigzag64,403,opt,name=F_Sint64_defaulted,json=fSint64Defaulted,def=-64" json:"F_Sint64_defaulted,omitempty"` - // Packed repeated fields (no string or bytes). - F_BoolRepeatedPacked []bool `protobuf:"varint,50,rep,packed,name=F_Bool_repeated_packed,json=fBoolRepeatedPacked" json:"F_Bool_repeated_packed,omitempty"` - F_Int32RepeatedPacked []int32 `protobuf:"varint,51,rep,packed,name=F_Int32_repeated_packed,json=fInt32RepeatedPacked" json:"F_Int32_repeated_packed,omitempty"` - F_Int64RepeatedPacked []int64 `protobuf:"varint,52,rep,packed,name=F_Int64_repeated_packed,json=fInt64RepeatedPacked" json:"F_Int64_repeated_packed,omitempty"` - F_Fixed32RepeatedPacked []uint32 `protobuf:"fixed32,53,rep,packed,name=F_Fixed32_repeated_packed,json=fFixed32RepeatedPacked" json:"F_Fixed32_repeated_packed,omitempty"` - F_Fixed64RepeatedPacked []uint64 `protobuf:"fixed64,54,rep,packed,name=F_Fixed64_repeated_packed,json=fFixed64RepeatedPacked" json:"F_Fixed64_repeated_packed,omitempty"` - F_Uint32RepeatedPacked []uint32 `protobuf:"varint,55,rep,packed,name=F_Uint32_repeated_packed,json=fUint32RepeatedPacked" json:"F_Uint32_repeated_packed,omitempty"` - F_Uint64RepeatedPacked []uint64 `protobuf:"varint,56,rep,packed,name=F_Uint64_repeated_packed,json=fUint64RepeatedPacked" json:"F_Uint64_repeated_packed,omitempty"` - F_FloatRepeatedPacked []float32 `protobuf:"fixed32,57,rep,packed,name=F_Float_repeated_packed,json=fFloatRepeatedPacked" json:"F_Float_repeated_packed,omitempty"` - F_DoubleRepeatedPacked []float64 `protobuf:"fixed64,58,rep,packed,name=F_Double_repeated_packed,json=fDoubleRepeatedPacked" json:"F_Double_repeated_packed,omitempty"` - F_Sint32RepeatedPacked []int32 `protobuf:"zigzag32,502,rep,packed,name=F_Sint32_repeated_packed,json=fSint32RepeatedPacked" json:"F_Sint32_repeated_packed,omitempty"` - F_Sint64RepeatedPacked []int64 `protobuf:"zigzag64,503,rep,packed,name=F_Sint64_repeated_packed,json=fSint64RepeatedPacked" json:"F_Sint64_repeated_packed,omitempty"` - Requiredgroup *GoTest_RequiredGroup `protobuf:"group,70,req,name=RequiredGroup,json=requiredgroup" json:"requiredgroup,omitempty"` - Repeatedgroup []*GoTest_RepeatedGroup `protobuf:"group,80,rep,name=RepeatedGroup,json=repeatedgroup" json:"repeatedgroup,omitempty"` - Optionalgroup *GoTest_OptionalGroup `protobuf:"group,90,opt,name=OptionalGroup,json=optionalgroup" json:"optionalgroup,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoTest) Reset() { *m = GoTest{} } -func (m *GoTest) String() string { return proto.CompactTextString(m) } -func (*GoTest) ProtoMessage() {} -func (*GoTest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } - -const Default_GoTest_F_BoolDefaulted bool = true -const Default_GoTest_F_Int32Defaulted int32 = 32 -const Default_GoTest_F_Int64Defaulted int64 = 64 -const Default_GoTest_F_Fixed32Defaulted uint32 = 320 -const Default_GoTest_F_Fixed64Defaulted uint64 = 640 -const Default_GoTest_F_Uint32Defaulted uint32 = 3200 -const Default_GoTest_F_Uint64Defaulted uint64 = 6400 -const Default_GoTest_F_FloatDefaulted float32 = 314159 -const Default_GoTest_F_DoubleDefaulted float64 = 271828 -const Default_GoTest_F_StringDefaulted string = "hello, \"world!\"\n" - -var Default_GoTest_F_BytesDefaulted []byte = []byte("Bignose") - -const Default_GoTest_F_Sint32Defaulted int32 = -32 -const Default_GoTest_F_Sint64Defaulted int64 = -64 - -func (m *GoTest) GetKind() GoTest_KIND { - if m != nil && m.Kind != nil { - return *m.Kind - } - return GoTest_VOID -} - -func (m *GoTest) GetTable() string { - if m != nil && m.Table != nil { - return *m.Table - } - return "" -} - -func (m *GoTest) GetParam() int32 { - if m != nil && m.Param != nil { - return *m.Param - } - return 0 -} - -func (m *GoTest) GetRequiredField() *GoTestField { - if m != nil { - return m.RequiredField - } - return nil -} - -func (m *GoTest) GetRepeatedField() []*GoTestField { - if m != nil { - return m.RepeatedField - } - return nil -} - -func (m *GoTest) GetOptionalField() *GoTestField { - if m != nil { - return m.OptionalField - } - return nil -} - -func (m *GoTest) GetF_BoolRequired() bool { - if m != nil && m.F_BoolRequired != nil { - return *m.F_BoolRequired - } - return false -} - -func (m *GoTest) GetF_Int32Required() int32 { - if m != nil && m.F_Int32Required != nil { - return *m.F_Int32Required - } - return 0 -} - -func (m *GoTest) GetF_Int64Required() int64 { - if m != nil && m.F_Int64Required != nil { - return *m.F_Int64Required - } - return 0 -} - -func (m *GoTest) GetF_Fixed32Required() uint32 { - if m != nil && m.F_Fixed32Required != nil { - return *m.F_Fixed32Required - } - return 0 -} - -func (m *GoTest) GetF_Fixed64Required() uint64 { - if m != nil && m.F_Fixed64Required != nil { - return *m.F_Fixed64Required - } - return 0 -} - -func (m *GoTest) GetF_Uint32Required() uint32 { - if m != nil && m.F_Uint32Required != nil { - return *m.F_Uint32Required - } - return 0 -} - -func (m *GoTest) GetF_Uint64Required() uint64 { - if m != nil && m.F_Uint64Required != nil { - return *m.F_Uint64Required - } - return 0 -} - -func (m *GoTest) GetF_FloatRequired() float32 { - if m != nil && m.F_FloatRequired != nil { - return *m.F_FloatRequired - } - return 0 -} - -func (m *GoTest) GetF_DoubleRequired() float64 { - if m != nil && m.F_DoubleRequired != nil { - return *m.F_DoubleRequired - } - return 0 -} - -func (m *GoTest) GetF_StringRequired() string { - if m != nil && m.F_StringRequired != nil { - return *m.F_StringRequired - } - return "" -} - -func (m *GoTest) GetF_BytesRequired() []byte { - if m != nil { - return m.F_BytesRequired - } - return nil -} - -func (m *GoTest) GetF_Sint32Required() int32 { - if m != nil && m.F_Sint32Required != nil { - return *m.F_Sint32Required - } - return 0 -} - -func (m *GoTest) GetF_Sint64Required() int64 { - if m != nil && m.F_Sint64Required != nil { - return *m.F_Sint64Required - } - return 0 -} - -func (m *GoTest) GetF_BoolRepeated() []bool { - if m != nil { - return m.F_BoolRepeated - } - return nil -} - -func (m *GoTest) GetF_Int32Repeated() []int32 { - if m != nil { - return m.F_Int32Repeated - } - return nil -} - -func (m *GoTest) GetF_Int64Repeated() []int64 { - if m != nil { - return m.F_Int64Repeated - } - return nil -} - -func (m *GoTest) GetF_Fixed32Repeated() []uint32 { - if m != nil { - return m.F_Fixed32Repeated - } - return nil -} - -func (m *GoTest) GetF_Fixed64Repeated() []uint64 { - if m != nil { - return m.F_Fixed64Repeated - } - return nil -} - -func (m *GoTest) GetF_Uint32Repeated() []uint32 { - if m != nil { - return m.F_Uint32Repeated - } - return nil -} - -func (m *GoTest) GetF_Uint64Repeated() []uint64 { - if m != nil { - return m.F_Uint64Repeated - } - return nil -} - -func (m *GoTest) GetF_FloatRepeated() []float32 { - if m != nil { - return m.F_FloatRepeated - } - return nil -} - -func (m *GoTest) GetF_DoubleRepeated() []float64 { - if m != nil { - return m.F_DoubleRepeated - } - return nil -} - -func (m *GoTest) GetF_StringRepeated() []string { - if m != nil { - return m.F_StringRepeated - } - return nil -} - -func (m *GoTest) GetF_BytesRepeated() [][]byte { - if m != nil { - return m.F_BytesRepeated - } - return nil -} - -func (m *GoTest) GetF_Sint32Repeated() []int32 { - if m != nil { - return m.F_Sint32Repeated - } - return nil -} - -func (m *GoTest) GetF_Sint64Repeated() []int64 { - if m != nil { - return m.F_Sint64Repeated - } - return nil -} - -func (m *GoTest) GetF_BoolOptional() bool { - if m != nil && m.F_BoolOptional != nil { - return *m.F_BoolOptional - } - return false -} - -func (m *GoTest) GetF_Int32Optional() int32 { - if m != nil && m.F_Int32Optional != nil { - return *m.F_Int32Optional - } - return 0 -} - -func (m *GoTest) GetF_Int64Optional() int64 { - if m != nil && m.F_Int64Optional != nil { - return *m.F_Int64Optional - } - return 0 -} - -func (m *GoTest) GetF_Fixed32Optional() uint32 { - if m != nil && m.F_Fixed32Optional != nil { - return *m.F_Fixed32Optional - } - return 0 -} - -func (m *GoTest) GetF_Fixed64Optional() uint64 { - if m != nil && m.F_Fixed64Optional != nil { - return *m.F_Fixed64Optional - } - return 0 -} - -func (m *GoTest) GetF_Uint32Optional() uint32 { - if m != nil && m.F_Uint32Optional != nil { - return *m.F_Uint32Optional - } - return 0 -} - -func (m *GoTest) GetF_Uint64Optional() uint64 { - if m != nil && m.F_Uint64Optional != nil { - return *m.F_Uint64Optional - } - return 0 -} - -func (m *GoTest) GetF_FloatOptional() float32 { - if m != nil && m.F_FloatOptional != nil { - return *m.F_FloatOptional - } - return 0 -} - -func (m *GoTest) GetF_DoubleOptional() float64 { - if m != nil && m.F_DoubleOptional != nil { - return *m.F_DoubleOptional - } - return 0 -} - -func (m *GoTest) GetF_StringOptional() string { - if m != nil && m.F_StringOptional != nil { - return *m.F_StringOptional - } - return "" -} - -func (m *GoTest) GetF_BytesOptional() []byte { - if m != nil { - return m.F_BytesOptional - } - return nil -} - -func (m *GoTest) GetF_Sint32Optional() int32 { - if m != nil && m.F_Sint32Optional != nil { - return *m.F_Sint32Optional - } - return 0 -} - -func (m *GoTest) GetF_Sint64Optional() int64 { - if m != nil && m.F_Sint64Optional != nil { - return *m.F_Sint64Optional - } - return 0 -} - -func (m *GoTest) GetF_BoolDefaulted() bool { - if m != nil && m.F_BoolDefaulted != nil { - return *m.F_BoolDefaulted - } - return Default_GoTest_F_BoolDefaulted -} - -func (m *GoTest) GetF_Int32Defaulted() int32 { - if m != nil && m.F_Int32Defaulted != nil { - return *m.F_Int32Defaulted - } - return Default_GoTest_F_Int32Defaulted -} - -func (m *GoTest) GetF_Int64Defaulted() int64 { - if m != nil && m.F_Int64Defaulted != nil { - return *m.F_Int64Defaulted - } - return Default_GoTest_F_Int64Defaulted -} - -func (m *GoTest) GetF_Fixed32Defaulted() uint32 { - if m != nil && m.F_Fixed32Defaulted != nil { - return *m.F_Fixed32Defaulted - } - return Default_GoTest_F_Fixed32Defaulted -} - -func (m *GoTest) GetF_Fixed64Defaulted() uint64 { - if m != nil && m.F_Fixed64Defaulted != nil { - return *m.F_Fixed64Defaulted - } - return Default_GoTest_F_Fixed64Defaulted -} - -func (m *GoTest) GetF_Uint32Defaulted() uint32 { - if m != nil && m.F_Uint32Defaulted != nil { - return *m.F_Uint32Defaulted - } - return Default_GoTest_F_Uint32Defaulted -} - -func (m *GoTest) GetF_Uint64Defaulted() uint64 { - if m != nil && m.F_Uint64Defaulted != nil { - return *m.F_Uint64Defaulted - } - return Default_GoTest_F_Uint64Defaulted -} - -func (m *GoTest) GetF_FloatDefaulted() float32 { - if m != nil && m.F_FloatDefaulted != nil { - return *m.F_FloatDefaulted - } - return Default_GoTest_F_FloatDefaulted -} - -func (m *GoTest) GetF_DoubleDefaulted() float64 { - if m != nil && m.F_DoubleDefaulted != nil { - return *m.F_DoubleDefaulted - } - return Default_GoTest_F_DoubleDefaulted -} - -func (m *GoTest) GetF_StringDefaulted() string { - if m != nil && m.F_StringDefaulted != nil { - return *m.F_StringDefaulted - } - return Default_GoTest_F_StringDefaulted -} - -func (m *GoTest) GetF_BytesDefaulted() []byte { - if m != nil && m.F_BytesDefaulted != nil { - return m.F_BytesDefaulted - } - return append([]byte(nil), Default_GoTest_F_BytesDefaulted...) -} - -func (m *GoTest) GetF_Sint32Defaulted() int32 { - if m != nil && m.F_Sint32Defaulted != nil { - return *m.F_Sint32Defaulted - } - return Default_GoTest_F_Sint32Defaulted -} - -func (m *GoTest) GetF_Sint64Defaulted() int64 { - if m != nil && m.F_Sint64Defaulted != nil { - return *m.F_Sint64Defaulted - } - return Default_GoTest_F_Sint64Defaulted -} - -func (m *GoTest) GetF_BoolRepeatedPacked() []bool { - if m != nil { - return m.F_BoolRepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Int32RepeatedPacked() []int32 { - if m != nil { - return m.F_Int32RepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Int64RepeatedPacked() []int64 { - if m != nil { - return m.F_Int64RepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Fixed32RepeatedPacked() []uint32 { - if m != nil { - return m.F_Fixed32RepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Fixed64RepeatedPacked() []uint64 { - if m != nil { - return m.F_Fixed64RepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Uint32RepeatedPacked() []uint32 { - if m != nil { - return m.F_Uint32RepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Uint64RepeatedPacked() []uint64 { - if m != nil { - return m.F_Uint64RepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_FloatRepeatedPacked() []float32 { - if m != nil { - return m.F_FloatRepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_DoubleRepeatedPacked() []float64 { - if m != nil { - return m.F_DoubleRepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Sint32RepeatedPacked() []int32 { - if m != nil { - return m.F_Sint32RepeatedPacked - } - return nil -} - -func (m *GoTest) GetF_Sint64RepeatedPacked() []int64 { - if m != nil { - return m.F_Sint64RepeatedPacked - } - return nil -} - -func (m *GoTest) GetRequiredgroup() *GoTest_RequiredGroup { - if m != nil { - return m.Requiredgroup - } - return nil -} - -func (m *GoTest) GetRepeatedgroup() []*GoTest_RepeatedGroup { - if m != nil { - return m.Repeatedgroup - } - return nil -} - -func (m *GoTest) GetOptionalgroup() *GoTest_OptionalGroup { - if m != nil { - return m.Optionalgroup - } - return nil -} - -// Required, repeated, and optional groups. -type GoTest_RequiredGroup struct { - RequiredField *string `protobuf:"bytes,71,req,name=RequiredField,json=requiredField" json:"RequiredField,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoTest_RequiredGroup) Reset() { *m = GoTest_RequiredGroup{} } -func (m *GoTest_RequiredGroup) String() string { return proto.CompactTextString(m) } -func (*GoTest_RequiredGroup) ProtoMessage() {} -func (*GoTest_RequiredGroup) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2, 0} } - -func (m *GoTest_RequiredGroup) GetRequiredField() string { - if m != nil && m.RequiredField != nil { - return *m.RequiredField - } - return "" -} - -type GoTest_RepeatedGroup struct { - RequiredField *string `protobuf:"bytes,81,req,name=RequiredField,json=requiredField" json:"RequiredField,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoTest_RepeatedGroup) Reset() { *m = GoTest_RepeatedGroup{} } -func (m *GoTest_RepeatedGroup) String() string { return proto.CompactTextString(m) } -func (*GoTest_RepeatedGroup) ProtoMessage() {} -func (*GoTest_RepeatedGroup) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2, 1} } - -func (m *GoTest_RepeatedGroup) GetRequiredField() string { - if m != nil && m.RequiredField != nil { - return *m.RequiredField - } - return "" -} - -type GoTest_OptionalGroup struct { - RequiredField *string `protobuf:"bytes,91,req,name=RequiredField,json=requiredField" json:"RequiredField,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoTest_OptionalGroup) Reset() { *m = GoTest_OptionalGroup{} } -func (m *GoTest_OptionalGroup) String() string { return proto.CompactTextString(m) } -func (*GoTest_OptionalGroup) ProtoMessage() {} -func (*GoTest_OptionalGroup) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2, 2} } - -func (m *GoTest_OptionalGroup) GetRequiredField() string { - if m != nil && m.RequiredField != nil { - return *m.RequiredField - } - return "" -} - -// For testing skipping of unrecognized fields. -// Numbers are all big, larger than tag numbers in GoTestField, -// the message used in the corresponding test. -type GoSkipTest struct { - SkipInt32 *int32 `protobuf:"varint,11,req,name=skip_int32,json=skipInt32" json:"skip_int32,omitempty"` - SkipFixed32 *uint32 `protobuf:"fixed32,12,req,name=skip_fixed32,json=skipFixed32" json:"skip_fixed32,omitempty"` - SkipFixed64 *uint64 `protobuf:"fixed64,13,req,name=skip_fixed64,json=skipFixed64" json:"skip_fixed64,omitempty"` - SkipString *string `protobuf:"bytes,14,req,name=skip_string,json=skipString" json:"skip_string,omitempty"` - Skipgroup *GoSkipTest_SkipGroup `protobuf:"group,15,req,name=SkipGroup,json=skipgroup" json:"skipgroup,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoSkipTest) Reset() { *m = GoSkipTest{} } -func (m *GoSkipTest) String() string { return proto.CompactTextString(m) } -func (*GoSkipTest) ProtoMessage() {} -func (*GoSkipTest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } - -func (m *GoSkipTest) GetSkipInt32() int32 { - if m != nil && m.SkipInt32 != nil { - return *m.SkipInt32 - } - return 0 -} - -func (m *GoSkipTest) GetSkipFixed32() uint32 { - if m != nil && m.SkipFixed32 != nil { - return *m.SkipFixed32 - } - return 0 -} - -func (m *GoSkipTest) GetSkipFixed64() uint64 { - if m != nil && m.SkipFixed64 != nil { - return *m.SkipFixed64 - } - return 0 -} - -func (m *GoSkipTest) GetSkipString() string { - if m != nil && m.SkipString != nil { - return *m.SkipString - } - return "" -} - -func (m *GoSkipTest) GetSkipgroup() *GoSkipTest_SkipGroup { - if m != nil { - return m.Skipgroup - } - return nil -} - -type GoSkipTest_SkipGroup struct { - GroupInt32 *int32 `protobuf:"varint,16,req,name=group_int32,json=groupInt32" json:"group_int32,omitempty"` - GroupString *string `protobuf:"bytes,17,req,name=group_string,json=groupString" json:"group_string,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GoSkipTest_SkipGroup) Reset() { *m = GoSkipTest_SkipGroup{} } -func (m *GoSkipTest_SkipGroup) String() string { return proto.CompactTextString(m) } -func (*GoSkipTest_SkipGroup) ProtoMessage() {} -func (*GoSkipTest_SkipGroup) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 0} } - -func (m *GoSkipTest_SkipGroup) GetGroupInt32() int32 { - if m != nil && m.GroupInt32 != nil { - return *m.GroupInt32 - } - return 0 -} - -func (m *GoSkipTest_SkipGroup) GetGroupString() string { - if m != nil && m.GroupString != nil { - return *m.GroupString - } - return "" -} - -// For testing packed/non-packed decoder switching. -// A serialized instance of one should be deserializable as the other. -type NonPackedTest struct { - A []int32 `protobuf:"varint,1,rep,name=a" json:"a,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *NonPackedTest) Reset() { *m = NonPackedTest{} } -func (m *NonPackedTest) String() string { return proto.CompactTextString(m) } -func (*NonPackedTest) ProtoMessage() {} -func (*NonPackedTest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } - -func (m *NonPackedTest) GetA() []int32 { - if m != nil { - return m.A - } - return nil -} - -type PackedTest struct { - B []int32 `protobuf:"varint,1,rep,packed,name=b" json:"b,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *PackedTest) Reset() { *m = PackedTest{} } -func (m *PackedTest) String() string { return proto.CompactTextString(m) } -func (*PackedTest) ProtoMessage() {} -func (*PackedTest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } - -func (m *PackedTest) GetB() []int32 { - if m != nil { - return m.B - } - return nil -} - -type MaxTag struct { - // Maximum possible tag number. - LastField *string `protobuf:"bytes,536870911,opt,name=last_field,json=lastField" json:"last_field,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MaxTag) Reset() { *m = MaxTag{} } -func (m *MaxTag) String() string { return proto.CompactTextString(m) } -func (*MaxTag) ProtoMessage() {} -func (*MaxTag) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } - -func (m *MaxTag) GetLastField() string { - if m != nil && m.LastField != nil { - return *m.LastField - } - return "" -} - -type OldMessage struct { - Nested *OldMessage_Nested `protobuf:"bytes,1,opt,name=nested" json:"nested,omitempty"` - Num *int32 `protobuf:"varint,2,opt,name=num" json:"num,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *OldMessage) Reset() { *m = OldMessage{} } -func (m *OldMessage) String() string { return proto.CompactTextString(m) } -func (*OldMessage) ProtoMessage() {} -func (*OldMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } - -func (m *OldMessage) GetNested() *OldMessage_Nested { - if m != nil { - return m.Nested - } - return nil -} - -func (m *OldMessage) GetNum() int32 { - if m != nil && m.Num != nil { - return *m.Num - } - return 0 -} - -type OldMessage_Nested struct { - Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *OldMessage_Nested) Reset() { *m = OldMessage_Nested{} } -func (m *OldMessage_Nested) String() string { return proto.CompactTextString(m) } -func (*OldMessage_Nested) ProtoMessage() {} -func (*OldMessage_Nested) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7, 0} } - -func (m *OldMessage_Nested) GetName() string { - if m != nil && m.Name != nil { - return *m.Name - } - return "" -} - -// NewMessage is wire compatible with OldMessage; -// imagine it as a future version. -type NewMessage struct { - Nested *NewMessage_Nested `protobuf:"bytes,1,opt,name=nested" json:"nested,omitempty"` - // This is an int32 in OldMessage. - Num *int64 `protobuf:"varint,2,opt,name=num" json:"num,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *NewMessage) Reset() { *m = NewMessage{} } -func (m *NewMessage) String() string { return proto.CompactTextString(m) } -func (*NewMessage) ProtoMessage() {} -func (*NewMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } - -func (m *NewMessage) GetNested() *NewMessage_Nested { - if m != nil { - return m.Nested - } - return nil -} - -func (m *NewMessage) GetNum() int64 { - if m != nil && m.Num != nil { - return *m.Num - } - return 0 -} - -type NewMessage_Nested struct { - Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - FoodGroup *string `protobuf:"bytes,2,opt,name=food_group,json=foodGroup" json:"food_group,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *NewMessage_Nested) Reset() { *m = NewMessage_Nested{} } -func (m *NewMessage_Nested) String() string { return proto.CompactTextString(m) } -func (*NewMessage_Nested) ProtoMessage() {} -func (*NewMessage_Nested) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8, 0} } - -func (m *NewMessage_Nested) GetName() string { - if m != nil && m.Name != nil { - return *m.Name - } - return "" -} - -func (m *NewMessage_Nested) GetFoodGroup() string { - if m != nil && m.FoodGroup != nil { - return *m.FoodGroup - } - return "" -} - -type InnerMessage struct { - Host *string `protobuf:"bytes,1,req,name=host" json:"host,omitempty"` - Port *int32 `protobuf:"varint,2,opt,name=port,def=4000" json:"port,omitempty"` - Connected *bool `protobuf:"varint,3,opt,name=connected" json:"connected,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *InnerMessage) Reset() { *m = InnerMessage{} } -func (m *InnerMessage) String() string { return proto.CompactTextString(m) } -func (*InnerMessage) ProtoMessage() {} -func (*InnerMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } - -const Default_InnerMessage_Port int32 = 4000 - -func (m *InnerMessage) GetHost() string { - if m != nil && m.Host != nil { - return *m.Host - } - return "" -} - -func (m *InnerMessage) GetPort() int32 { - if m != nil && m.Port != nil { - return *m.Port - } - return Default_InnerMessage_Port -} - -func (m *InnerMessage) GetConnected() bool { - if m != nil && m.Connected != nil { - return *m.Connected - } - return false -} - -type OtherMessage struct { - Key *int64 `protobuf:"varint,1,opt,name=key" json:"key,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value" json:"value,omitempty"` - Weight *float32 `protobuf:"fixed32,3,opt,name=weight" json:"weight,omitempty"` - Inner *InnerMessage `protobuf:"bytes,4,opt,name=inner" json:"inner,omitempty"` - XXX_extensions map[int32]proto.Extension `json:"-"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *OtherMessage) Reset() { *m = OtherMessage{} } -func (m *OtherMessage) String() string { return proto.CompactTextString(m) } -func (*OtherMessage) ProtoMessage() {} -func (*OtherMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } - -var extRange_OtherMessage = []proto.ExtensionRange{ - {100, 536870911}, -} - -func (*OtherMessage) ExtensionRangeArray() []proto.ExtensionRange { - return extRange_OtherMessage -} -func (m *OtherMessage) ExtensionMap() map[int32]proto.Extension { - if m.XXX_extensions == nil { - m.XXX_extensions = make(map[int32]proto.Extension) - } - return m.XXX_extensions -} - -func (m *OtherMessage) GetKey() int64 { - if m != nil && m.Key != nil { - return *m.Key - } - return 0 -} - -func (m *OtherMessage) GetValue() []byte { - if m != nil { - return m.Value - } - return nil -} - -func (m *OtherMessage) GetWeight() float32 { - if m != nil && m.Weight != nil { - return *m.Weight - } - return 0 -} - -func (m *OtherMessage) GetInner() *InnerMessage { - if m != nil { - return m.Inner - } - return nil -} - -type MyMessage struct { - Count *int32 `protobuf:"varint,1,req,name=count" json:"count,omitempty"` - Name *string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` - Quote *string `protobuf:"bytes,3,opt,name=quote" json:"quote,omitempty"` - Pet []string `protobuf:"bytes,4,rep,name=pet" json:"pet,omitempty"` - Inner *InnerMessage `protobuf:"bytes,5,opt,name=inner" json:"inner,omitempty"` - Others []*OtherMessage `protobuf:"bytes,6,rep,name=others" json:"others,omitempty"` - RepInner []*InnerMessage `protobuf:"bytes,12,rep,name=rep_inner,json=repInner" json:"rep_inner,omitempty"` - Bikeshed *MyMessage_Color `protobuf:"varint,7,opt,name=bikeshed,enum=testdata.MyMessage_Color" json:"bikeshed,omitempty"` - Somegroup *MyMessage_SomeGroup `protobuf:"group,8,opt,name=SomeGroup,json=somegroup" json:"somegroup,omitempty"` - // This field becomes [][]byte in the generated code. - RepBytes [][]byte `protobuf:"bytes,10,rep,name=rep_bytes,json=repBytes" json:"rep_bytes,omitempty"` - Bigfloat *float64 `protobuf:"fixed64,11,opt,name=bigfloat" json:"bigfloat,omitempty"` - XXX_extensions map[int32]proto.Extension `json:"-"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MyMessage) Reset() { *m = MyMessage{} } -func (m *MyMessage) String() string { return proto.CompactTextString(m) } -func (*MyMessage) ProtoMessage() {} -func (*MyMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } - -var extRange_MyMessage = []proto.ExtensionRange{ - {100, 536870911}, -} - -func (*MyMessage) ExtensionRangeArray() []proto.ExtensionRange { - return extRange_MyMessage -} -func (m *MyMessage) ExtensionMap() map[int32]proto.Extension { - if m.XXX_extensions == nil { - m.XXX_extensions = make(map[int32]proto.Extension) - } - return m.XXX_extensions -} - -func (m *MyMessage) GetCount() int32 { - if m != nil && m.Count != nil { - return *m.Count - } - return 0 -} - -func (m *MyMessage) GetName() string { - if m != nil && m.Name != nil { - return *m.Name - } - return "" -} - -func (m *MyMessage) GetQuote() string { - if m != nil && m.Quote != nil { - return *m.Quote - } - return "" -} - -func (m *MyMessage) GetPet() []string { - if m != nil { - return m.Pet - } - return nil -} - -func (m *MyMessage) GetInner() *InnerMessage { - if m != nil { - return m.Inner - } - return nil -} - -func (m *MyMessage) GetOthers() []*OtherMessage { - if m != nil { - return m.Others - } - return nil -} - -func (m *MyMessage) GetRepInner() []*InnerMessage { - if m != nil { - return m.RepInner - } - return nil -} - -func (m *MyMessage) GetBikeshed() MyMessage_Color { - if m != nil && m.Bikeshed != nil { - return *m.Bikeshed - } - return MyMessage_RED -} - -func (m *MyMessage) GetSomegroup() *MyMessage_SomeGroup { - if m != nil { - return m.Somegroup - } - return nil -} - -func (m *MyMessage) GetRepBytes() [][]byte { - if m != nil { - return m.RepBytes - } - return nil -} - -func (m *MyMessage) GetBigfloat() float64 { - if m != nil && m.Bigfloat != nil { - return *m.Bigfloat - } - return 0 -} - -type MyMessage_SomeGroup struct { - GroupField *int32 `protobuf:"varint,9,opt,name=group_field,json=groupField" json:"group_field,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MyMessage_SomeGroup) Reset() { *m = MyMessage_SomeGroup{} } -func (m *MyMessage_SomeGroup) String() string { return proto.CompactTextString(m) } -func (*MyMessage_SomeGroup) ProtoMessage() {} -func (*MyMessage_SomeGroup) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11, 0} } - -func (m *MyMessage_SomeGroup) GetGroupField() int32 { - if m != nil && m.GroupField != nil { - return *m.GroupField - } - return 0 -} - -type Ext struct { - Data *string `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Ext) Reset() { *m = Ext{} } -func (m *Ext) String() string { return proto.CompactTextString(m) } -func (*Ext) ProtoMessage() {} -func (*Ext) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } - -func (m *Ext) GetData() string { - if m != nil && m.Data != nil { - return *m.Data - } - return "" -} - -var E_Ext_More = &proto.ExtensionDesc{ - ExtendedType: (*MyMessage)(nil), - ExtensionType: (*Ext)(nil), - Field: 103, - Name: "testdata.Ext.more", - Tag: "bytes,103,opt,name=more", -} - -var E_Ext_Text = &proto.ExtensionDesc{ - ExtendedType: (*MyMessage)(nil), - ExtensionType: (*string)(nil), - Field: 104, - Name: "testdata.Ext.text", - Tag: "bytes,104,opt,name=text", -} - -var E_Ext_Number = &proto.ExtensionDesc{ - ExtendedType: (*MyMessage)(nil), - ExtensionType: (*int32)(nil), - Field: 105, - Name: "testdata.Ext.number", - Tag: "varint,105,opt,name=number", -} - -type ComplexExtension struct { - First *int32 `protobuf:"varint,1,opt,name=first" json:"first,omitempty"` - Second *int32 `protobuf:"varint,2,opt,name=second" json:"second,omitempty"` - Third []int32 `protobuf:"varint,3,rep,name=third" json:"third,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *ComplexExtension) Reset() { *m = ComplexExtension{} } -func (m *ComplexExtension) String() string { return proto.CompactTextString(m) } -func (*ComplexExtension) ProtoMessage() {} -func (*ComplexExtension) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } - -func (m *ComplexExtension) GetFirst() int32 { - if m != nil && m.First != nil { - return *m.First - } - return 0 -} - -func (m *ComplexExtension) GetSecond() int32 { - if m != nil && m.Second != nil { - return *m.Second - } - return 0 -} - -func (m *ComplexExtension) GetThird() []int32 { - if m != nil { - return m.Third - } - return nil -} - -type DefaultsMessage struct { - XXX_extensions map[int32]proto.Extension `json:"-"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *DefaultsMessage) Reset() { *m = DefaultsMessage{} } -func (m *DefaultsMessage) String() string { return proto.CompactTextString(m) } -func (*DefaultsMessage) ProtoMessage() {} -func (*DefaultsMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } - -var extRange_DefaultsMessage = []proto.ExtensionRange{ - {100, 536870911}, -} - -func (*DefaultsMessage) ExtensionRangeArray() []proto.ExtensionRange { - return extRange_DefaultsMessage -} -func (m *DefaultsMessage) ExtensionMap() map[int32]proto.Extension { - if m.XXX_extensions == nil { - m.XXX_extensions = make(map[int32]proto.Extension) - } - return m.XXX_extensions -} - -type MyMessageSet struct { - XXX_extensions map[int32]proto.Extension `json:"-"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MyMessageSet) Reset() { *m = MyMessageSet{} } -func (m *MyMessageSet) String() string { return proto.CompactTextString(m) } -func (*MyMessageSet) ProtoMessage() {} -func (*MyMessageSet) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } - -func (m *MyMessageSet) Marshal() ([]byte, error) { - return proto.MarshalMessageSet(m.ExtensionMap()) -} -func (m *MyMessageSet) Unmarshal(buf []byte) error { - return proto.UnmarshalMessageSet(buf, m.ExtensionMap()) -} -func (m *MyMessageSet) MarshalJSON() ([]byte, error) { - return proto.MarshalMessageSetJSON(m.XXX_extensions) -} -func (m *MyMessageSet) UnmarshalJSON(buf []byte) error { - return proto.UnmarshalMessageSetJSON(buf, m.XXX_extensions) -} - -// ensure MyMessageSet satisfies proto.Marshaler and proto.Unmarshaler -var _ proto.Marshaler = (*MyMessageSet)(nil) -var _ proto.Unmarshaler = (*MyMessageSet)(nil) - -var extRange_MyMessageSet = []proto.ExtensionRange{ - {100, 2147483646}, -} - -func (*MyMessageSet) ExtensionRangeArray() []proto.ExtensionRange { - return extRange_MyMessageSet -} -func (m *MyMessageSet) ExtensionMap() map[int32]proto.Extension { - if m.XXX_extensions == nil { - m.XXX_extensions = make(map[int32]proto.Extension) - } - return m.XXX_extensions -} - -type Empty struct { - XXX_unrecognized []byte `json:"-"` -} - -func (m *Empty) Reset() { *m = Empty{} } -func (m *Empty) String() string { return proto.CompactTextString(m) } -func (*Empty) ProtoMessage() {} -func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } - -type MessageList struct { - Message []*MessageList_Message `protobuf:"group,1,rep,name=Message,json=message" json:"message,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MessageList) Reset() { *m = MessageList{} } -func (m *MessageList) String() string { return proto.CompactTextString(m) } -func (*MessageList) ProtoMessage() {} -func (*MessageList) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } - -func (m *MessageList) GetMessage() []*MessageList_Message { - if m != nil { - return m.Message - } - return nil -} - -type MessageList_Message struct { - Name *string `protobuf:"bytes,2,req,name=name" json:"name,omitempty"` - Count *int32 `protobuf:"varint,3,req,name=count" json:"count,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MessageList_Message) Reset() { *m = MessageList_Message{} } -func (m *MessageList_Message) String() string { return proto.CompactTextString(m) } -func (*MessageList_Message) ProtoMessage() {} -func (*MessageList_Message) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17, 0} } - -func (m *MessageList_Message) GetName() string { - if m != nil && m.Name != nil { - return *m.Name - } - return "" -} - -func (m *MessageList_Message) GetCount() int32 { - if m != nil && m.Count != nil { - return *m.Count - } - return 0 -} - -type Strings struct { - StringField *string `protobuf:"bytes,1,opt,name=string_field,json=stringField" json:"string_field,omitempty"` - BytesField []byte `protobuf:"bytes,2,opt,name=bytes_field,json=bytesField" json:"bytes_field,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Strings) Reset() { *m = Strings{} } -func (m *Strings) String() string { return proto.CompactTextString(m) } -func (*Strings) ProtoMessage() {} -func (*Strings) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } - -func (m *Strings) GetStringField() string { - if m != nil && m.StringField != nil { - return *m.StringField - } - return "" -} - -func (m *Strings) GetBytesField() []byte { - if m != nil { - return m.BytesField - } - return nil -} - -type Defaults struct { - // Default-valued fields of all basic types. - // Same as GoTest, but copied here to make testing easier. - F_Bool *bool `protobuf:"varint,1,opt,name=F_Bool,json=fBool,def=1" json:"F_Bool,omitempty"` - F_Int32 *int32 `protobuf:"varint,2,opt,name=F_Int32,json=fInt32,def=32" json:"F_Int32,omitempty"` - F_Int64 *int64 `protobuf:"varint,3,opt,name=F_Int64,json=fInt64,def=64" json:"F_Int64,omitempty"` - F_Fixed32 *uint32 `protobuf:"fixed32,4,opt,name=F_Fixed32,json=fFixed32,def=320" json:"F_Fixed32,omitempty"` - F_Fixed64 *uint64 `protobuf:"fixed64,5,opt,name=F_Fixed64,json=fFixed64,def=640" json:"F_Fixed64,omitempty"` - F_Uint32 *uint32 `protobuf:"varint,6,opt,name=F_Uint32,json=fUint32,def=3200" json:"F_Uint32,omitempty"` - F_Uint64 *uint64 `protobuf:"varint,7,opt,name=F_Uint64,json=fUint64,def=6400" json:"F_Uint64,omitempty"` - F_Float *float32 `protobuf:"fixed32,8,opt,name=F_Float,json=fFloat,def=314159" json:"F_Float,omitempty"` - F_Double *float64 `protobuf:"fixed64,9,opt,name=F_Double,json=fDouble,def=271828" json:"F_Double,omitempty"` - F_String *string `protobuf:"bytes,10,opt,name=F_String,json=fString,def=hello, \"world!\"\n" json:"F_String,omitempty"` - F_Bytes []byte `protobuf:"bytes,11,opt,name=F_Bytes,json=fBytes,def=Bignose" json:"F_Bytes,omitempty"` - F_Sint32 *int32 `protobuf:"zigzag32,12,opt,name=F_Sint32,json=fSint32,def=-32" json:"F_Sint32,omitempty"` - F_Sint64 *int64 `protobuf:"zigzag64,13,opt,name=F_Sint64,json=fSint64,def=-64" json:"F_Sint64,omitempty"` - F_Enum *Defaults_Color `protobuf:"varint,14,opt,name=F_Enum,json=fEnum,enum=testdata.Defaults_Color,def=1" json:"F_Enum,omitempty"` - // More fields with crazy defaults. - F_Pinf *float32 `protobuf:"fixed32,15,opt,name=F_Pinf,json=fPinf,def=inf" json:"F_Pinf,omitempty"` - F_Ninf *float32 `protobuf:"fixed32,16,opt,name=F_Ninf,json=fNinf,def=-inf" json:"F_Ninf,omitempty"` - F_Nan *float32 `protobuf:"fixed32,17,opt,name=F_Nan,json=fNan,def=nan" json:"F_Nan,omitempty"` - // Sub-message. - Sub *SubDefaults `protobuf:"bytes,18,opt,name=sub" json:"sub,omitempty"` - // Redundant but explicit defaults. - StrZero *string `protobuf:"bytes,19,opt,name=str_zero,json=strZero,def=" json:"str_zero,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Defaults) Reset() { *m = Defaults{} } -func (m *Defaults) String() string { return proto.CompactTextString(m) } -func (*Defaults) ProtoMessage() {} -func (*Defaults) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} } - -const Default_Defaults_F_Bool bool = true -const Default_Defaults_F_Int32 int32 = 32 -const Default_Defaults_F_Int64 int64 = 64 -const Default_Defaults_F_Fixed32 uint32 = 320 -const Default_Defaults_F_Fixed64 uint64 = 640 -const Default_Defaults_F_Uint32 uint32 = 3200 -const Default_Defaults_F_Uint64 uint64 = 6400 -const Default_Defaults_F_Float float32 = 314159 -const Default_Defaults_F_Double float64 = 271828 -const Default_Defaults_F_String string = "hello, \"world!\"\n" - -var Default_Defaults_F_Bytes []byte = []byte("Bignose") - -const Default_Defaults_F_Sint32 int32 = -32 -const Default_Defaults_F_Sint64 int64 = -64 -const Default_Defaults_F_Enum Defaults_Color = Defaults_GREEN - -var Default_Defaults_F_Pinf float32 = float32(math.Inf(1)) -var Default_Defaults_F_Ninf float32 = float32(math.Inf(-1)) -var Default_Defaults_F_Nan float32 = float32(math.NaN()) - -func (m *Defaults) GetF_Bool() bool { - if m != nil && m.F_Bool != nil { - return *m.F_Bool - } - return Default_Defaults_F_Bool -} - -func (m *Defaults) GetF_Int32() int32 { - if m != nil && m.F_Int32 != nil { - return *m.F_Int32 - } - return Default_Defaults_F_Int32 -} - -func (m *Defaults) GetF_Int64() int64 { - if m != nil && m.F_Int64 != nil { - return *m.F_Int64 - } - return Default_Defaults_F_Int64 -} - -func (m *Defaults) GetF_Fixed32() uint32 { - if m != nil && m.F_Fixed32 != nil { - return *m.F_Fixed32 - } - return Default_Defaults_F_Fixed32 -} - -func (m *Defaults) GetF_Fixed64() uint64 { - if m != nil && m.F_Fixed64 != nil { - return *m.F_Fixed64 - } - return Default_Defaults_F_Fixed64 -} - -func (m *Defaults) GetF_Uint32() uint32 { - if m != nil && m.F_Uint32 != nil { - return *m.F_Uint32 - } - return Default_Defaults_F_Uint32 -} - -func (m *Defaults) GetF_Uint64() uint64 { - if m != nil && m.F_Uint64 != nil { - return *m.F_Uint64 - } - return Default_Defaults_F_Uint64 -} - -func (m *Defaults) GetF_Float() float32 { - if m != nil && m.F_Float != nil { - return *m.F_Float - } - return Default_Defaults_F_Float -} - -func (m *Defaults) GetF_Double() float64 { - if m != nil && m.F_Double != nil { - return *m.F_Double - } - return Default_Defaults_F_Double -} - -func (m *Defaults) GetF_String() string { - if m != nil && m.F_String != nil { - return *m.F_String - } - return Default_Defaults_F_String -} - -func (m *Defaults) GetF_Bytes() []byte { - if m != nil && m.F_Bytes != nil { - return m.F_Bytes - } - return append([]byte(nil), Default_Defaults_F_Bytes...) -} - -func (m *Defaults) GetF_Sint32() int32 { - if m != nil && m.F_Sint32 != nil { - return *m.F_Sint32 - } - return Default_Defaults_F_Sint32 -} - -func (m *Defaults) GetF_Sint64() int64 { - if m != nil && m.F_Sint64 != nil { - return *m.F_Sint64 - } - return Default_Defaults_F_Sint64 -} - -func (m *Defaults) GetF_Enum() Defaults_Color { - if m != nil && m.F_Enum != nil { - return *m.F_Enum - } - return Default_Defaults_F_Enum -} - -func (m *Defaults) GetF_Pinf() float32 { - if m != nil && m.F_Pinf != nil { - return *m.F_Pinf - } - return Default_Defaults_F_Pinf -} - -func (m *Defaults) GetF_Ninf() float32 { - if m != nil && m.F_Ninf != nil { - return *m.F_Ninf - } - return Default_Defaults_F_Ninf -} - -func (m *Defaults) GetF_Nan() float32 { - if m != nil && m.F_Nan != nil { - return *m.F_Nan - } - return Default_Defaults_F_Nan -} - -func (m *Defaults) GetSub() *SubDefaults { - if m != nil { - return m.Sub - } - return nil -} - -func (m *Defaults) GetStrZero() string { - if m != nil && m.StrZero != nil { - return *m.StrZero - } - return "" -} - -type SubDefaults struct { - N *int64 `protobuf:"varint,1,opt,name=n,def=7" json:"n,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *SubDefaults) Reset() { *m = SubDefaults{} } -func (m *SubDefaults) String() string { return proto.CompactTextString(m) } -func (*SubDefaults) ProtoMessage() {} -func (*SubDefaults) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} } - -const Default_SubDefaults_N int64 = 7 - -func (m *SubDefaults) GetN() int64 { - if m != nil && m.N != nil { - return *m.N - } - return Default_SubDefaults_N -} - -type RepeatedEnum struct { - Color []RepeatedEnum_Color `protobuf:"varint,1,rep,name=color,enum=testdata.RepeatedEnum_Color" json:"color,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *RepeatedEnum) Reset() { *m = RepeatedEnum{} } -func (m *RepeatedEnum) String() string { return proto.CompactTextString(m) } -func (*RepeatedEnum) ProtoMessage() {} -func (*RepeatedEnum) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} } - -func (m *RepeatedEnum) GetColor() []RepeatedEnum_Color { - if m != nil { - return m.Color - } - return nil -} - -type MoreRepeated struct { - Bools []bool `protobuf:"varint,1,rep,name=bools" json:"bools,omitempty"` - BoolsPacked []bool `protobuf:"varint,2,rep,packed,name=bools_packed,json=boolsPacked" json:"bools_packed,omitempty"` - Ints []int32 `protobuf:"varint,3,rep,name=ints" json:"ints,omitempty"` - IntsPacked []int32 `protobuf:"varint,4,rep,packed,name=ints_packed,json=intsPacked" json:"ints_packed,omitempty"` - Int64SPacked []int64 `protobuf:"varint,7,rep,packed,name=int64s_packed,json=int64sPacked" json:"int64s_packed,omitempty"` - Strings []string `protobuf:"bytes,5,rep,name=strings" json:"strings,omitempty"` - Fixeds []uint32 `protobuf:"fixed32,6,rep,name=fixeds" json:"fixeds,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MoreRepeated) Reset() { *m = MoreRepeated{} } -func (m *MoreRepeated) String() string { return proto.CompactTextString(m) } -func (*MoreRepeated) ProtoMessage() {} -func (*MoreRepeated) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} } - -func (m *MoreRepeated) GetBools() []bool { - if m != nil { - return m.Bools - } - return nil -} - -func (m *MoreRepeated) GetBoolsPacked() []bool { - if m != nil { - return m.BoolsPacked - } - return nil -} - -func (m *MoreRepeated) GetInts() []int32 { - if m != nil { - return m.Ints - } - return nil -} - -func (m *MoreRepeated) GetIntsPacked() []int32 { - if m != nil { - return m.IntsPacked - } - return nil -} - -func (m *MoreRepeated) GetInt64SPacked() []int64 { - if m != nil { - return m.Int64SPacked - } - return nil -} - -func (m *MoreRepeated) GetStrings() []string { - if m != nil { - return m.Strings - } - return nil -} - -func (m *MoreRepeated) GetFixeds() []uint32 { - if m != nil { - return m.Fixeds - } - return nil -} - -type GroupOld struct { - G *GroupOld_G `protobuf:"group,101,opt,name=G,json=g" json:"g,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GroupOld) Reset() { *m = GroupOld{} } -func (m *GroupOld) String() string { return proto.CompactTextString(m) } -func (*GroupOld) ProtoMessage() {} -func (*GroupOld) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} } - -func (m *GroupOld) GetG() *GroupOld_G { - if m != nil { - return m.G - } - return nil -} - -type GroupOld_G struct { - X *int32 `protobuf:"varint,2,opt,name=x" json:"x,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GroupOld_G) Reset() { *m = GroupOld_G{} } -func (m *GroupOld_G) String() string { return proto.CompactTextString(m) } -func (*GroupOld_G) ProtoMessage() {} -func (*GroupOld_G) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23, 0} } - -func (m *GroupOld_G) GetX() int32 { - if m != nil && m.X != nil { - return *m.X - } - return 0 -} - -type GroupNew struct { - G *GroupNew_G `protobuf:"group,101,opt,name=G,json=g" json:"g,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GroupNew) Reset() { *m = GroupNew{} } -func (m *GroupNew) String() string { return proto.CompactTextString(m) } -func (*GroupNew) ProtoMessage() {} -func (*GroupNew) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} } - -func (m *GroupNew) GetG() *GroupNew_G { - if m != nil { - return m.G - } - return nil -} - -type GroupNew_G struct { - X *int32 `protobuf:"varint,2,opt,name=x" json:"x,omitempty"` - Y *int32 `protobuf:"varint,3,opt,name=y" json:"y,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *GroupNew_G) Reset() { *m = GroupNew_G{} } -func (m *GroupNew_G) String() string { return proto.CompactTextString(m) } -func (*GroupNew_G) ProtoMessage() {} -func (*GroupNew_G) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24, 0} } - -func (m *GroupNew_G) GetX() int32 { - if m != nil && m.X != nil { - return *m.X - } - return 0 -} - -func (m *GroupNew_G) GetY() int32 { - if m != nil && m.Y != nil { - return *m.Y - } - return 0 -} - -type FloatingPoint struct { - F *float64 `protobuf:"fixed64,1,req,name=f" json:"f,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *FloatingPoint) Reset() { *m = FloatingPoint{} } -func (m *FloatingPoint) String() string { return proto.CompactTextString(m) } -func (*FloatingPoint) ProtoMessage() {} -func (*FloatingPoint) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} } - -func (m *FloatingPoint) GetF() float64 { - if m != nil && m.F != nil { - return *m.F - } - return 0 -} - -type MessageWithMap struct { - NameMapping map[int32]string `protobuf:"bytes,1,rep,name=name_mapping,json=nameMapping" json:"name_mapping,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - MsgMapping map[int64]*FloatingPoint `protobuf:"bytes,2,rep,name=msg_mapping,json=msgMapping" json:"msg_mapping,omitempty" protobuf_key:"zigzag64,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - ByteMapping map[bool][]byte `protobuf:"bytes,3,rep,name=byte_mapping,json=byteMapping" json:"byte_mapping,omitempty" protobuf_key:"varint,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - StrToStr map[string]string `protobuf:"bytes,4,rep,name=str_to_str,json=strToStr" json:"str_to_str,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *MessageWithMap) Reset() { *m = MessageWithMap{} } -func (m *MessageWithMap) String() string { return proto.CompactTextString(m) } -func (*MessageWithMap) ProtoMessage() {} -func (*MessageWithMap) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} } - -func (m *MessageWithMap) GetNameMapping() map[int32]string { - if m != nil { - return m.NameMapping - } - return nil -} - -func (m *MessageWithMap) GetMsgMapping() map[int64]*FloatingPoint { - if m != nil { - return m.MsgMapping - } - return nil -} - -func (m *MessageWithMap) GetByteMapping() map[bool][]byte { - if m != nil { - return m.ByteMapping - } - return nil -} - -func (m *MessageWithMap) GetStrToStr() map[string]string { - if m != nil { - return m.StrToStr - } - return nil -} - -type Oneof struct { - // Types that are valid to be assigned to Union: - // *Oneof_F_Bool - // *Oneof_F_Int32 - // *Oneof_F_Int64 - // *Oneof_F_Fixed32 - // *Oneof_F_Fixed64 - // *Oneof_F_Uint32 - // *Oneof_F_Uint64 - // *Oneof_F_Float - // *Oneof_F_Double - // *Oneof_F_String - // *Oneof_F_Bytes - // *Oneof_F_Sint32 - // *Oneof_F_Sint64 - // *Oneof_F_Enum - // *Oneof_F_Message - // *Oneof_FGroup - // *Oneof_F_Largest_Tag - Union isOneof_Union `protobuf_oneof:"union"` - // Types that are valid to be assigned to Tormato: - // *Oneof_Value - Tormato isOneof_Tormato `protobuf_oneof:"tormato"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Oneof) Reset() { *m = Oneof{} } -func (m *Oneof) String() string { return proto.CompactTextString(m) } -func (*Oneof) ProtoMessage() {} -func (*Oneof) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} } - -type isOneof_Union interface { - isOneof_Union() -} -type isOneof_Tormato interface { - isOneof_Tormato() -} - -type Oneof_F_Bool struct { - F_Bool bool `protobuf:"varint,1,opt,name=F_Bool,json=fBool,oneof"` -} -type Oneof_F_Int32 struct { - F_Int32 int32 `protobuf:"varint,2,opt,name=F_Int32,json=fInt32,oneof"` -} -type Oneof_F_Int64 struct { - F_Int64 int64 `protobuf:"varint,3,opt,name=F_Int64,json=fInt64,oneof"` -} -type Oneof_F_Fixed32 struct { - F_Fixed32 uint32 `protobuf:"fixed32,4,opt,name=F_Fixed32,json=fFixed32,oneof"` -} -type Oneof_F_Fixed64 struct { - F_Fixed64 uint64 `protobuf:"fixed64,5,opt,name=F_Fixed64,json=fFixed64,oneof"` -} -type Oneof_F_Uint32 struct { - F_Uint32 uint32 `protobuf:"varint,6,opt,name=F_Uint32,json=fUint32,oneof"` -} -type Oneof_F_Uint64 struct { - F_Uint64 uint64 `protobuf:"varint,7,opt,name=F_Uint64,json=fUint64,oneof"` -} -type Oneof_F_Float struct { - F_Float float32 `protobuf:"fixed32,8,opt,name=F_Float,json=fFloat,oneof"` -} -type Oneof_F_Double struct { - F_Double float64 `protobuf:"fixed64,9,opt,name=F_Double,json=fDouble,oneof"` -} -type Oneof_F_String struct { - F_String string `protobuf:"bytes,10,opt,name=F_String,json=fString,oneof"` -} -type Oneof_F_Bytes struct { - F_Bytes []byte `protobuf:"bytes,11,opt,name=F_Bytes,json=fBytes,oneof"` -} -type Oneof_F_Sint32 struct { - F_Sint32 int32 `protobuf:"zigzag32,12,opt,name=F_Sint32,json=fSint32,oneof"` -} -type Oneof_F_Sint64 struct { - F_Sint64 int64 `protobuf:"zigzag64,13,opt,name=F_Sint64,json=fSint64,oneof"` -} -type Oneof_F_Enum struct { - F_Enum MyMessage_Color `protobuf:"varint,14,opt,name=F_Enum,json=fEnum,enum=testdata.MyMessage_Color,oneof"` -} -type Oneof_F_Message struct { - F_Message *GoTestField `protobuf:"bytes,15,opt,name=F_Message,json=fMessage,oneof"` -} -type Oneof_FGroup struct { - FGroup *Oneof_F_Group `protobuf:"group,16,opt,name=F_Group,json=fGroup,oneof"` -} -type Oneof_F_Largest_Tag struct { - F_Largest_Tag int32 `protobuf:"varint,536870911,opt,name=F_Largest_Tag,json=fLargestTag,oneof"` -} -type Oneof_Value struct { - Value int32 `protobuf:"varint,100,opt,name=value,oneof"` -} - -func (*Oneof_F_Bool) isOneof_Union() {} -func (*Oneof_F_Int32) isOneof_Union() {} -func (*Oneof_F_Int64) isOneof_Union() {} -func (*Oneof_F_Fixed32) isOneof_Union() {} -func (*Oneof_F_Fixed64) isOneof_Union() {} -func (*Oneof_F_Uint32) isOneof_Union() {} -func (*Oneof_F_Uint64) isOneof_Union() {} -func (*Oneof_F_Float) isOneof_Union() {} -func (*Oneof_F_Double) isOneof_Union() {} -func (*Oneof_F_String) isOneof_Union() {} -func (*Oneof_F_Bytes) isOneof_Union() {} -func (*Oneof_F_Sint32) isOneof_Union() {} -func (*Oneof_F_Sint64) isOneof_Union() {} -func (*Oneof_F_Enum) isOneof_Union() {} -func (*Oneof_F_Message) isOneof_Union() {} -func (*Oneof_FGroup) isOneof_Union() {} -func (*Oneof_F_Largest_Tag) isOneof_Union() {} -func (*Oneof_Value) isOneof_Tormato() {} - -func (m *Oneof) GetUnion() isOneof_Union { - if m != nil { - return m.Union - } - return nil -} -func (m *Oneof) GetTormato() isOneof_Tormato { - if m != nil { - return m.Tormato - } - return nil -} - -func (m *Oneof) GetF_Bool() bool { - if x, ok := m.GetUnion().(*Oneof_F_Bool); ok { - return x.F_Bool - } - return false -} - -func (m *Oneof) GetF_Int32() int32 { - if x, ok := m.GetUnion().(*Oneof_F_Int32); ok { - return x.F_Int32 - } - return 0 -} - -func (m *Oneof) GetF_Int64() int64 { - if x, ok := m.GetUnion().(*Oneof_F_Int64); ok { - return x.F_Int64 - } - return 0 -} - -func (m *Oneof) GetF_Fixed32() uint32 { - if x, ok := m.GetUnion().(*Oneof_F_Fixed32); ok { - return x.F_Fixed32 - } - return 0 -} - -func (m *Oneof) GetF_Fixed64() uint64 { - if x, ok := m.GetUnion().(*Oneof_F_Fixed64); ok { - return x.F_Fixed64 - } - return 0 -} - -func (m *Oneof) GetF_Uint32() uint32 { - if x, ok := m.GetUnion().(*Oneof_F_Uint32); ok { - return x.F_Uint32 - } - return 0 -} - -func (m *Oneof) GetF_Uint64() uint64 { - if x, ok := m.GetUnion().(*Oneof_F_Uint64); ok { - return x.F_Uint64 - } - return 0 -} - -func (m *Oneof) GetF_Float() float32 { - if x, ok := m.GetUnion().(*Oneof_F_Float); ok { - return x.F_Float - } - return 0 -} - -func (m *Oneof) GetF_Double() float64 { - if x, ok := m.GetUnion().(*Oneof_F_Double); ok { - return x.F_Double - } - return 0 -} - -func (m *Oneof) GetF_String() string { - if x, ok := m.GetUnion().(*Oneof_F_String); ok { - return x.F_String - } - return "" -} - -func (m *Oneof) GetF_Bytes() []byte { - if x, ok := m.GetUnion().(*Oneof_F_Bytes); ok { - return x.F_Bytes - } - return nil -} - -func (m *Oneof) GetF_Sint32() int32 { - if x, ok := m.GetUnion().(*Oneof_F_Sint32); ok { - return x.F_Sint32 - } - return 0 -} - -func (m *Oneof) GetF_Sint64() int64 { - if x, ok := m.GetUnion().(*Oneof_F_Sint64); ok { - return x.F_Sint64 - } - return 0 -} - -func (m *Oneof) GetF_Enum() MyMessage_Color { - if x, ok := m.GetUnion().(*Oneof_F_Enum); ok { - return x.F_Enum - } - return MyMessage_RED -} - -func (m *Oneof) GetF_Message() *GoTestField { - if x, ok := m.GetUnion().(*Oneof_F_Message); ok { - return x.F_Message - } - return nil -} - -func (m *Oneof) GetFGroup() *Oneof_F_Group { - if x, ok := m.GetUnion().(*Oneof_FGroup); ok { - return x.FGroup - } - return nil -} - -func (m *Oneof) GetF_Largest_Tag() int32 { - if x, ok := m.GetUnion().(*Oneof_F_Largest_Tag); ok { - return x.F_Largest_Tag - } - return 0 -} - -func (m *Oneof) GetValue() int32 { - if x, ok := m.GetTormato().(*Oneof_Value); ok { - return x.Value - } - return 0 -} - -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Oneof) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Oneof_OneofMarshaler, _Oneof_OneofUnmarshaler, _Oneof_OneofSizer, []interface{}{ - (*Oneof_F_Bool)(nil), - (*Oneof_F_Int32)(nil), - (*Oneof_F_Int64)(nil), - (*Oneof_F_Fixed32)(nil), - (*Oneof_F_Fixed64)(nil), - (*Oneof_F_Uint32)(nil), - (*Oneof_F_Uint64)(nil), - (*Oneof_F_Float)(nil), - (*Oneof_F_Double)(nil), - (*Oneof_F_String)(nil), - (*Oneof_F_Bytes)(nil), - (*Oneof_F_Sint32)(nil), - (*Oneof_F_Sint64)(nil), - (*Oneof_F_Enum)(nil), - (*Oneof_F_Message)(nil), - (*Oneof_FGroup)(nil), - (*Oneof_F_Largest_Tag)(nil), - (*Oneof_Value)(nil), - } -} - -func _Oneof_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Oneof) - // union - switch x := m.Union.(type) { - case *Oneof_F_Bool: - t := uint64(0) - if x.F_Bool { - t = 1 - } - b.EncodeVarint(1<<3 | proto.WireVarint) - b.EncodeVarint(t) - case *Oneof_F_Int32: - b.EncodeVarint(2<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.F_Int32)) - case *Oneof_F_Int64: - b.EncodeVarint(3<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.F_Int64)) - case *Oneof_F_Fixed32: - b.EncodeVarint(4<<3 | proto.WireFixed32) - b.EncodeFixed32(uint64(x.F_Fixed32)) - case *Oneof_F_Fixed64: - b.EncodeVarint(5<<3 | proto.WireFixed64) - b.EncodeFixed64(uint64(x.F_Fixed64)) - case *Oneof_F_Uint32: - b.EncodeVarint(6<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.F_Uint32)) - case *Oneof_F_Uint64: - b.EncodeVarint(7<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.F_Uint64)) - case *Oneof_F_Float: - b.EncodeVarint(8<<3 | proto.WireFixed32) - b.EncodeFixed32(uint64(math.Float32bits(x.F_Float))) - case *Oneof_F_Double: - b.EncodeVarint(9<<3 | proto.WireFixed64) - b.EncodeFixed64(math.Float64bits(x.F_Double)) - case *Oneof_F_String: - b.EncodeVarint(10<<3 | proto.WireBytes) - b.EncodeStringBytes(x.F_String) - case *Oneof_F_Bytes: - b.EncodeVarint(11<<3 | proto.WireBytes) - b.EncodeRawBytes(x.F_Bytes) - case *Oneof_F_Sint32: - b.EncodeVarint(12<<3 | proto.WireVarint) - b.EncodeZigzag32(uint64(x.F_Sint32)) - case *Oneof_F_Sint64: - b.EncodeVarint(13<<3 | proto.WireVarint) - b.EncodeZigzag64(uint64(x.F_Sint64)) - case *Oneof_F_Enum: - b.EncodeVarint(14<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.F_Enum)) - case *Oneof_F_Message: - b.EncodeVarint(15<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.F_Message); err != nil { - return err - } - case *Oneof_FGroup: - b.EncodeVarint(16<<3 | proto.WireStartGroup) - if err := b.Marshal(x.FGroup); err != nil { - return err - } - b.EncodeVarint(16<<3 | proto.WireEndGroup) - case *Oneof_F_Largest_Tag: - b.EncodeVarint(536870911<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.F_Largest_Tag)) - case nil: - default: - return fmt.Errorf("Oneof.Union has unexpected type %T", x) - } - // tormato - switch x := m.Tormato.(type) { - case *Oneof_Value: - b.EncodeVarint(100<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.Value)) - case nil: - default: - return fmt.Errorf("Oneof.Tormato has unexpected type %T", x) - } - return nil -} - -func _Oneof_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Oneof) - switch tag { - case 1: // union.F_Bool - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Oneof_F_Bool{x != 0} - return true, err - case 2: // union.F_Int32 - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Oneof_F_Int32{int32(x)} - return true, err - case 3: // union.F_Int64 - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Oneof_F_Int64{int64(x)} - return true, err - case 4: // union.F_Fixed32 - if wire != proto.WireFixed32 { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeFixed32() - m.Union = &Oneof_F_Fixed32{uint32(x)} - return true, err - case 5: // union.F_Fixed64 - if wire != proto.WireFixed64 { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeFixed64() - m.Union = &Oneof_F_Fixed64{x} - return true, err - case 6: // union.F_Uint32 - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Oneof_F_Uint32{uint32(x)} - return true, err - case 7: // union.F_Uint64 - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Oneof_F_Uint64{x} - return true, err - case 8: // union.F_Float - if wire != proto.WireFixed32 { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeFixed32() - m.Union = &Oneof_F_Float{math.Float32frombits(uint32(x))} - return true, err - case 9: // union.F_Double - if wire != proto.WireFixed64 { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeFixed64() - m.Union = &Oneof_F_Double{math.Float64frombits(x)} - return true, err - case 10: // union.F_String - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Union = &Oneof_F_String{x} - return true, err - case 11: // union.F_Bytes - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeRawBytes(true) - m.Union = &Oneof_F_Bytes{x} - return true, err - case 12: // union.F_Sint32 - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeZigzag32() - m.Union = &Oneof_F_Sint32{int32(x)} - return true, err - case 13: // union.F_Sint64 - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeZigzag64() - m.Union = &Oneof_F_Sint64{int64(x)} - return true, err - case 14: // union.F_Enum - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Oneof_F_Enum{MyMessage_Color(x)} - return true, err - case 15: // union.F_Message - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(GoTestField) - err := b.DecodeMessage(msg) - m.Union = &Oneof_F_Message{msg} - return true, err - case 16: // union.f_group - if wire != proto.WireStartGroup { - return true, proto.ErrInternalBadWireType - } - msg := new(Oneof_F_Group) - err := b.DecodeGroup(msg) - m.Union = &Oneof_FGroup{msg} - return true, err - case 536870911: // union.F_Largest_Tag - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Oneof_F_Largest_Tag{int32(x)} - return true, err - case 100: // tormato.value - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Tormato = &Oneof_Value{int32(x)} - return true, err - default: - return false, nil - } -} - -func _Oneof_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Oneof) - // union - switch x := m.Union.(type) { - case *Oneof_F_Bool: - n += proto.SizeVarint(1<<3 | proto.WireVarint) - n += 1 - case *Oneof_F_Int32: - n += proto.SizeVarint(2<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.F_Int32)) - case *Oneof_F_Int64: - n += proto.SizeVarint(3<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.F_Int64)) - case *Oneof_F_Fixed32: - n += proto.SizeVarint(4<<3 | proto.WireFixed32) - n += 4 - case *Oneof_F_Fixed64: - n += proto.SizeVarint(5<<3 | proto.WireFixed64) - n += 8 - case *Oneof_F_Uint32: - n += proto.SizeVarint(6<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.F_Uint32)) - case *Oneof_F_Uint64: - n += proto.SizeVarint(7<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.F_Uint64)) - case *Oneof_F_Float: - n += proto.SizeVarint(8<<3 | proto.WireFixed32) - n += 4 - case *Oneof_F_Double: - n += proto.SizeVarint(9<<3 | proto.WireFixed64) - n += 8 - case *Oneof_F_String: - n += proto.SizeVarint(10<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(len(x.F_String))) - n += len(x.F_String) - case *Oneof_F_Bytes: - n += proto.SizeVarint(11<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(len(x.F_Bytes))) - n += len(x.F_Bytes) - case *Oneof_F_Sint32: - n += proto.SizeVarint(12<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64((uint32(x.F_Sint32) << 1) ^ uint32((int32(x.F_Sint32) >> 31)))) - case *Oneof_F_Sint64: - n += proto.SizeVarint(13<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(uint64(x.F_Sint64<<1) ^ uint64((int64(x.F_Sint64) >> 63)))) - case *Oneof_F_Enum: - n += proto.SizeVarint(14<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.F_Enum)) - case *Oneof_F_Message: - s := proto.Size(x.F_Message) - n += proto.SizeVarint(15<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case *Oneof_FGroup: - n += proto.SizeVarint(16<<3 | proto.WireStartGroup) - n += proto.Size(x.FGroup) - n += proto.SizeVarint(16<<3 | proto.WireEndGroup) - case *Oneof_F_Largest_Tag: - n += proto.SizeVarint(536870911<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.F_Largest_Tag)) - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - // tormato - switch x := m.Tormato.(type) { - case *Oneof_Value: - n += proto.SizeVarint(100<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.Value)) - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - -type Oneof_F_Group struct { - X *int32 `protobuf:"varint,17,opt,name=x" json:"x,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Oneof_F_Group) Reset() { *m = Oneof_F_Group{} } -func (m *Oneof_F_Group) String() string { return proto.CompactTextString(m) } -func (*Oneof_F_Group) ProtoMessage() {} -func (*Oneof_F_Group) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27, 0} } - -func (m *Oneof_F_Group) GetX() int32 { - if m != nil && m.X != nil { - return *m.X - } - return 0 -} - -type Communique struct { - MakeMeCry *bool `protobuf:"varint,1,opt,name=make_me_cry,json=makeMeCry" json:"make_me_cry,omitempty"` - // This is a oneof, called "union". - // - // Types that are valid to be assigned to Union: - // *Communique_Number - // *Communique_Name - // *Communique_Data - // *Communique_TempC - // *Communique_Col - // *Communique_Msg - Union isCommunique_Union `protobuf_oneof:"union"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Communique) Reset() { *m = Communique{} } -func (m *Communique) String() string { return proto.CompactTextString(m) } -func (*Communique) ProtoMessage() {} -func (*Communique) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} } - -type isCommunique_Union interface { - isCommunique_Union() -} - -type Communique_Number struct { - Number int32 `protobuf:"varint,5,opt,name=number,oneof"` -} -type Communique_Name struct { - Name string `protobuf:"bytes,6,opt,name=name,oneof"` -} -type Communique_Data struct { - Data []byte `protobuf:"bytes,7,opt,name=data,oneof"` -} -type Communique_TempC struct { - TempC float64 `protobuf:"fixed64,8,opt,name=temp_c,json=tempC,oneof"` -} -type Communique_Col struct { - Col MyMessage_Color `protobuf:"varint,9,opt,name=col,enum=testdata.MyMessage_Color,oneof"` -} -type Communique_Msg struct { - Msg *Strings `protobuf:"bytes,10,opt,name=msg,oneof"` -} - -func (*Communique_Number) isCommunique_Union() {} -func (*Communique_Name) isCommunique_Union() {} -func (*Communique_Data) isCommunique_Union() {} -func (*Communique_TempC) isCommunique_Union() {} -func (*Communique_Col) isCommunique_Union() {} -func (*Communique_Msg) isCommunique_Union() {} - -func (m *Communique) GetUnion() isCommunique_Union { - if m != nil { - return m.Union - } - return nil -} - -func (m *Communique) GetMakeMeCry() bool { - if m != nil && m.MakeMeCry != nil { - return *m.MakeMeCry - } - return false -} - -func (m *Communique) GetNumber() int32 { - if x, ok := m.GetUnion().(*Communique_Number); ok { - return x.Number - } - return 0 -} - -func (m *Communique) GetName() string { - if x, ok := m.GetUnion().(*Communique_Name); ok { - return x.Name - } - return "" -} - -func (m *Communique) GetData() []byte { - if x, ok := m.GetUnion().(*Communique_Data); ok { - return x.Data - } - return nil -} - -func (m *Communique) GetTempC() float64 { - if x, ok := m.GetUnion().(*Communique_TempC); ok { - return x.TempC - } - return 0 -} - -func (m *Communique) GetCol() MyMessage_Color { - if x, ok := m.GetUnion().(*Communique_Col); ok { - return x.Col - } - return MyMessage_RED -} - -func (m *Communique) GetMsg() *Strings { - if x, ok := m.GetUnion().(*Communique_Msg); ok { - return x.Msg - } - return nil -} - -// XXX_OneofFuncs is for the internal use of the proto package. -func (*Communique) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { - return _Communique_OneofMarshaler, _Communique_OneofUnmarshaler, _Communique_OneofSizer, []interface{}{ - (*Communique_Number)(nil), - (*Communique_Name)(nil), - (*Communique_Data)(nil), - (*Communique_TempC)(nil), - (*Communique_Col)(nil), - (*Communique_Msg)(nil), - } -} - -func _Communique_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { - m := msg.(*Communique) - // union - switch x := m.Union.(type) { - case *Communique_Number: - b.EncodeVarint(5<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.Number)) - case *Communique_Name: - b.EncodeVarint(6<<3 | proto.WireBytes) - b.EncodeStringBytes(x.Name) - case *Communique_Data: - b.EncodeVarint(7<<3 | proto.WireBytes) - b.EncodeRawBytes(x.Data) - case *Communique_TempC: - b.EncodeVarint(8<<3 | proto.WireFixed64) - b.EncodeFixed64(math.Float64bits(x.TempC)) - case *Communique_Col: - b.EncodeVarint(9<<3 | proto.WireVarint) - b.EncodeVarint(uint64(x.Col)) - case *Communique_Msg: - b.EncodeVarint(10<<3 | proto.WireBytes) - if err := b.EncodeMessage(x.Msg); err != nil { - return err - } - case nil: - default: - return fmt.Errorf("Communique.Union has unexpected type %T", x) - } - return nil -} - -func _Communique_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { - m := msg.(*Communique) - switch tag { - case 5: // union.number - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Communique_Number{int32(x)} - return true, err - case 6: // union.name - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeStringBytes() - m.Union = &Communique_Name{x} - return true, err - case 7: // union.data - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeRawBytes(true) - m.Union = &Communique_Data{x} - return true, err - case 8: // union.temp_c - if wire != proto.WireFixed64 { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeFixed64() - m.Union = &Communique_TempC{math.Float64frombits(x)} - return true, err - case 9: // union.col - if wire != proto.WireVarint { - return true, proto.ErrInternalBadWireType - } - x, err := b.DecodeVarint() - m.Union = &Communique_Col{MyMessage_Color(x)} - return true, err - case 10: // union.msg - if wire != proto.WireBytes { - return true, proto.ErrInternalBadWireType - } - msg := new(Strings) - err := b.DecodeMessage(msg) - m.Union = &Communique_Msg{msg} - return true, err - default: - return false, nil - } -} - -func _Communique_OneofSizer(msg proto.Message) (n int) { - m := msg.(*Communique) - // union - switch x := m.Union.(type) { - case *Communique_Number: - n += proto.SizeVarint(5<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.Number)) - case *Communique_Name: - n += proto.SizeVarint(6<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(len(x.Name))) - n += len(x.Name) - case *Communique_Data: - n += proto.SizeVarint(7<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(len(x.Data))) - n += len(x.Data) - case *Communique_TempC: - n += proto.SizeVarint(8<<3 | proto.WireFixed64) - n += 8 - case *Communique_Col: - n += proto.SizeVarint(9<<3 | proto.WireVarint) - n += proto.SizeVarint(uint64(x.Col)) - case *Communique_Msg: - s := proto.Size(x.Msg) - n += proto.SizeVarint(10<<3 | proto.WireBytes) - n += proto.SizeVarint(uint64(s)) - n += s - case nil: - default: - panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) - } - return n -} - -var E_Greeting = &proto.ExtensionDesc{ - ExtendedType: (*MyMessage)(nil), - ExtensionType: ([]string)(nil), - Field: 106, - Name: "testdata.greeting", - Tag: "bytes,106,rep,name=greeting", -} - -var E_Complex = &proto.ExtensionDesc{ - ExtendedType: (*OtherMessage)(nil), - ExtensionType: (*ComplexExtension)(nil), - Field: 200, - Name: "testdata.complex", - Tag: "bytes,200,opt,name=complex", -} - -var E_RComplex = &proto.ExtensionDesc{ - ExtendedType: (*OtherMessage)(nil), - ExtensionType: ([]*ComplexExtension)(nil), - Field: 201, - Name: "testdata.r_complex", - Tag: "bytes,201,rep,name=r_complex,json=rComplex", -} - -var E_NoDefaultDouble = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*float64)(nil), - Field: 101, - Name: "testdata.no_default_double", - Tag: "fixed64,101,opt,name=no_default_double,json=noDefaultDouble", -} - -var E_NoDefaultFloat = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*float32)(nil), - Field: 102, - Name: "testdata.no_default_float", - Tag: "fixed32,102,opt,name=no_default_float,json=noDefaultFloat", -} - -var E_NoDefaultInt32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int32)(nil), - Field: 103, - Name: "testdata.no_default_int32", - Tag: "varint,103,opt,name=no_default_int32,json=noDefaultInt32", -} - -var E_NoDefaultInt64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int64)(nil), - Field: 104, - Name: "testdata.no_default_int64", - Tag: "varint,104,opt,name=no_default_int64,json=noDefaultInt64", -} - -var E_NoDefaultUint32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint32)(nil), - Field: 105, - Name: "testdata.no_default_uint32", - Tag: "varint,105,opt,name=no_default_uint32,json=noDefaultUint32", -} - -var E_NoDefaultUint64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint64)(nil), - Field: 106, - Name: "testdata.no_default_uint64", - Tag: "varint,106,opt,name=no_default_uint64,json=noDefaultUint64", -} - -var E_NoDefaultSint32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int32)(nil), - Field: 107, - Name: "testdata.no_default_sint32", - Tag: "zigzag32,107,opt,name=no_default_sint32,json=noDefaultSint32", -} - -var E_NoDefaultSint64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int64)(nil), - Field: 108, - Name: "testdata.no_default_sint64", - Tag: "zigzag64,108,opt,name=no_default_sint64,json=noDefaultSint64", -} - -var E_NoDefaultFixed32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint32)(nil), - Field: 109, - Name: "testdata.no_default_fixed32", - Tag: "fixed32,109,opt,name=no_default_fixed32,json=noDefaultFixed32", -} - -var E_NoDefaultFixed64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint64)(nil), - Field: 110, - Name: "testdata.no_default_fixed64", - Tag: "fixed64,110,opt,name=no_default_fixed64,json=noDefaultFixed64", -} - -var E_NoDefaultSfixed32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int32)(nil), - Field: 111, - Name: "testdata.no_default_sfixed32", - Tag: "fixed32,111,opt,name=no_default_sfixed32,json=noDefaultSfixed32", -} - -var E_NoDefaultSfixed64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int64)(nil), - Field: 112, - Name: "testdata.no_default_sfixed64", - Tag: "fixed64,112,opt,name=no_default_sfixed64,json=noDefaultSfixed64", -} - -var E_NoDefaultBool = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*bool)(nil), - Field: 113, - Name: "testdata.no_default_bool", - Tag: "varint,113,opt,name=no_default_bool,json=noDefaultBool", -} - -var E_NoDefaultString = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*string)(nil), - Field: 114, - Name: "testdata.no_default_string", - Tag: "bytes,114,opt,name=no_default_string,json=noDefaultString", -} - -var E_NoDefaultBytes = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: ([]byte)(nil), - Field: 115, - Name: "testdata.no_default_bytes", - Tag: "bytes,115,opt,name=no_default_bytes,json=noDefaultBytes", -} - -var E_NoDefaultEnum = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*DefaultsMessage_DefaultsEnum)(nil), - Field: 116, - Name: "testdata.no_default_enum", - Tag: "varint,116,opt,name=no_default_enum,json=noDefaultEnum,enum=testdata.DefaultsMessage_DefaultsEnum", -} - -var E_DefaultDouble = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*float64)(nil), - Field: 201, - Name: "testdata.default_double", - Tag: "fixed64,201,opt,name=default_double,json=defaultDouble,def=3.1415", -} - -var E_DefaultFloat = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*float32)(nil), - Field: 202, - Name: "testdata.default_float", - Tag: "fixed32,202,opt,name=default_float,json=defaultFloat,def=3.14", -} - -var E_DefaultInt32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int32)(nil), - Field: 203, - Name: "testdata.default_int32", - Tag: "varint,203,opt,name=default_int32,json=defaultInt32,def=42", -} - -var E_DefaultInt64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int64)(nil), - Field: 204, - Name: "testdata.default_int64", - Tag: "varint,204,opt,name=default_int64,json=defaultInt64,def=43", -} - -var E_DefaultUint32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint32)(nil), - Field: 205, - Name: "testdata.default_uint32", - Tag: "varint,205,opt,name=default_uint32,json=defaultUint32,def=44", -} - -var E_DefaultUint64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint64)(nil), - Field: 206, - Name: "testdata.default_uint64", - Tag: "varint,206,opt,name=default_uint64,json=defaultUint64,def=45", -} - -var E_DefaultSint32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int32)(nil), - Field: 207, - Name: "testdata.default_sint32", - Tag: "zigzag32,207,opt,name=default_sint32,json=defaultSint32,def=46", -} - -var E_DefaultSint64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int64)(nil), - Field: 208, - Name: "testdata.default_sint64", - Tag: "zigzag64,208,opt,name=default_sint64,json=defaultSint64,def=47", -} - -var E_DefaultFixed32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint32)(nil), - Field: 209, - Name: "testdata.default_fixed32", - Tag: "fixed32,209,opt,name=default_fixed32,json=defaultFixed32,def=48", -} - -var E_DefaultFixed64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*uint64)(nil), - Field: 210, - Name: "testdata.default_fixed64", - Tag: "fixed64,210,opt,name=default_fixed64,json=defaultFixed64,def=49", -} - -var E_DefaultSfixed32 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int32)(nil), - Field: 211, - Name: "testdata.default_sfixed32", - Tag: "fixed32,211,opt,name=default_sfixed32,json=defaultSfixed32,def=50", -} - -var E_DefaultSfixed64 = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*int64)(nil), - Field: 212, - Name: "testdata.default_sfixed64", - Tag: "fixed64,212,opt,name=default_sfixed64,json=defaultSfixed64,def=51", -} - -var E_DefaultBool = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*bool)(nil), - Field: 213, - Name: "testdata.default_bool", - Tag: "varint,213,opt,name=default_bool,json=defaultBool,def=1", -} - -var E_DefaultString = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*string)(nil), - Field: 214, - Name: "testdata.default_string", - Tag: "bytes,214,opt,name=default_string,json=defaultString,def=Hello, string", -} - -var E_DefaultBytes = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: ([]byte)(nil), - Field: 215, - Name: "testdata.default_bytes", - Tag: "bytes,215,opt,name=default_bytes,json=defaultBytes,def=Hello, bytes", -} - -var E_DefaultEnum = &proto.ExtensionDesc{ - ExtendedType: (*DefaultsMessage)(nil), - ExtensionType: (*DefaultsMessage_DefaultsEnum)(nil), - Field: 216, - Name: "testdata.default_enum", - Tag: "varint,216,opt,name=default_enum,json=defaultEnum,enum=testdata.DefaultsMessage_DefaultsEnum,def=1", -} - -var E_X201 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 201, - Name: "testdata.x201", - Tag: "bytes,201,opt,name=x201", -} - -var E_X202 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 202, - Name: "testdata.x202", - Tag: "bytes,202,opt,name=x202", -} - -var E_X203 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 203, - Name: "testdata.x203", - Tag: "bytes,203,opt,name=x203", -} - -var E_X204 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 204, - Name: "testdata.x204", - Tag: "bytes,204,opt,name=x204", -} - -var E_X205 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 205, - Name: "testdata.x205", - Tag: "bytes,205,opt,name=x205", -} - -var E_X206 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 206, - Name: "testdata.x206", - Tag: "bytes,206,opt,name=x206", -} - -var E_X207 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 207, - Name: "testdata.x207", - Tag: "bytes,207,opt,name=x207", -} - -var E_X208 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 208, - Name: "testdata.x208", - Tag: "bytes,208,opt,name=x208", -} - -var E_X209 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 209, - Name: "testdata.x209", - Tag: "bytes,209,opt,name=x209", -} - -var E_X210 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 210, - Name: "testdata.x210", - Tag: "bytes,210,opt,name=x210", -} - -var E_X211 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 211, - Name: "testdata.x211", - Tag: "bytes,211,opt,name=x211", -} - -var E_X212 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 212, - Name: "testdata.x212", - Tag: "bytes,212,opt,name=x212", -} - -var E_X213 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 213, - Name: "testdata.x213", - Tag: "bytes,213,opt,name=x213", -} - -var E_X214 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 214, - Name: "testdata.x214", - Tag: "bytes,214,opt,name=x214", -} - -var E_X215 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 215, - Name: "testdata.x215", - Tag: "bytes,215,opt,name=x215", -} - -var E_X216 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 216, - Name: "testdata.x216", - Tag: "bytes,216,opt,name=x216", -} - -var E_X217 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 217, - Name: "testdata.x217", - Tag: "bytes,217,opt,name=x217", -} - -var E_X218 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 218, - Name: "testdata.x218", - Tag: "bytes,218,opt,name=x218", -} - -var E_X219 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 219, - Name: "testdata.x219", - Tag: "bytes,219,opt,name=x219", -} - -var E_X220 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 220, - Name: "testdata.x220", - Tag: "bytes,220,opt,name=x220", -} - -var E_X221 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 221, - Name: "testdata.x221", - Tag: "bytes,221,opt,name=x221", -} - -var E_X222 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 222, - Name: "testdata.x222", - Tag: "bytes,222,opt,name=x222", -} - -var E_X223 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 223, - Name: "testdata.x223", - Tag: "bytes,223,opt,name=x223", -} - -var E_X224 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 224, - Name: "testdata.x224", - Tag: "bytes,224,opt,name=x224", -} - -var E_X225 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 225, - Name: "testdata.x225", - Tag: "bytes,225,opt,name=x225", -} - -var E_X226 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 226, - Name: "testdata.x226", - Tag: "bytes,226,opt,name=x226", -} - -var E_X227 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 227, - Name: "testdata.x227", - Tag: "bytes,227,opt,name=x227", -} - -var E_X228 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 228, - Name: "testdata.x228", - Tag: "bytes,228,opt,name=x228", -} - -var E_X229 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 229, - Name: "testdata.x229", - Tag: "bytes,229,opt,name=x229", -} - -var E_X230 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 230, - Name: "testdata.x230", - Tag: "bytes,230,opt,name=x230", -} - -var E_X231 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 231, - Name: "testdata.x231", - Tag: "bytes,231,opt,name=x231", -} - -var E_X232 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 232, - Name: "testdata.x232", - Tag: "bytes,232,opt,name=x232", -} - -var E_X233 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 233, - Name: "testdata.x233", - Tag: "bytes,233,opt,name=x233", -} - -var E_X234 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 234, - Name: "testdata.x234", - Tag: "bytes,234,opt,name=x234", -} - -var E_X235 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 235, - Name: "testdata.x235", - Tag: "bytes,235,opt,name=x235", -} - -var E_X236 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 236, - Name: "testdata.x236", - Tag: "bytes,236,opt,name=x236", -} - -var E_X237 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 237, - Name: "testdata.x237", - Tag: "bytes,237,opt,name=x237", -} - -var E_X238 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 238, - Name: "testdata.x238", - Tag: "bytes,238,opt,name=x238", -} - -var E_X239 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 239, - Name: "testdata.x239", - Tag: "bytes,239,opt,name=x239", -} - -var E_X240 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 240, - Name: "testdata.x240", - Tag: "bytes,240,opt,name=x240", -} - -var E_X241 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 241, - Name: "testdata.x241", - Tag: "bytes,241,opt,name=x241", -} - -var E_X242 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 242, - Name: "testdata.x242", - Tag: "bytes,242,opt,name=x242", -} - -var E_X243 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 243, - Name: "testdata.x243", - Tag: "bytes,243,opt,name=x243", -} - -var E_X244 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 244, - Name: "testdata.x244", - Tag: "bytes,244,opt,name=x244", -} - -var E_X245 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 245, - Name: "testdata.x245", - Tag: "bytes,245,opt,name=x245", -} - -var E_X246 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 246, - Name: "testdata.x246", - Tag: "bytes,246,opt,name=x246", -} - -var E_X247 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 247, - Name: "testdata.x247", - Tag: "bytes,247,opt,name=x247", -} - -var E_X248 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 248, - Name: "testdata.x248", - Tag: "bytes,248,opt,name=x248", -} - -var E_X249 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 249, - Name: "testdata.x249", - Tag: "bytes,249,opt,name=x249", -} - -var E_X250 = &proto.ExtensionDesc{ - ExtendedType: (*MyMessageSet)(nil), - ExtensionType: (*Empty)(nil), - Field: 250, - Name: "testdata.x250", - Tag: "bytes,250,opt,name=x250", -} - -func init() { - proto.RegisterType((*GoEnum)(nil), "testdata.GoEnum") - proto.RegisterType((*GoTestField)(nil), "testdata.GoTestField") - proto.RegisterType((*GoTest)(nil), "testdata.GoTest") - proto.RegisterType((*GoTest_RequiredGroup)(nil), "testdata.GoTest.RequiredGroup") - proto.RegisterType((*GoTest_RepeatedGroup)(nil), "testdata.GoTest.RepeatedGroup") - proto.RegisterType((*GoTest_OptionalGroup)(nil), "testdata.GoTest.OptionalGroup") - proto.RegisterType((*GoSkipTest)(nil), "testdata.GoSkipTest") - proto.RegisterType((*GoSkipTest_SkipGroup)(nil), "testdata.GoSkipTest.SkipGroup") - proto.RegisterType((*NonPackedTest)(nil), "testdata.NonPackedTest") - proto.RegisterType((*PackedTest)(nil), "testdata.PackedTest") - proto.RegisterType((*MaxTag)(nil), "testdata.MaxTag") - proto.RegisterType((*OldMessage)(nil), "testdata.OldMessage") - proto.RegisterType((*OldMessage_Nested)(nil), "testdata.OldMessage.Nested") - proto.RegisterType((*NewMessage)(nil), "testdata.NewMessage") - proto.RegisterType((*NewMessage_Nested)(nil), "testdata.NewMessage.Nested") - proto.RegisterType((*InnerMessage)(nil), "testdata.InnerMessage") - proto.RegisterType((*OtherMessage)(nil), "testdata.OtherMessage") - proto.RegisterType((*MyMessage)(nil), "testdata.MyMessage") - proto.RegisterType((*MyMessage_SomeGroup)(nil), "testdata.MyMessage.SomeGroup") - proto.RegisterType((*Ext)(nil), "testdata.Ext") - proto.RegisterType((*ComplexExtension)(nil), "testdata.ComplexExtension") - proto.RegisterType((*DefaultsMessage)(nil), "testdata.DefaultsMessage") - proto.RegisterType((*MyMessageSet)(nil), "testdata.MyMessageSet") - proto.RegisterType((*Empty)(nil), "testdata.Empty") - proto.RegisterType((*MessageList)(nil), "testdata.MessageList") - proto.RegisterType((*MessageList_Message)(nil), "testdata.MessageList.Message") - proto.RegisterType((*Strings)(nil), "testdata.Strings") - proto.RegisterType((*Defaults)(nil), "testdata.Defaults") - proto.RegisterType((*SubDefaults)(nil), "testdata.SubDefaults") - proto.RegisterType((*RepeatedEnum)(nil), "testdata.RepeatedEnum") - proto.RegisterType((*MoreRepeated)(nil), "testdata.MoreRepeated") - proto.RegisterType((*GroupOld)(nil), "testdata.GroupOld") - proto.RegisterType((*GroupOld_G)(nil), "testdata.GroupOld.G") - proto.RegisterType((*GroupNew)(nil), "testdata.GroupNew") - proto.RegisterType((*GroupNew_G)(nil), "testdata.GroupNew.G") - proto.RegisterType((*FloatingPoint)(nil), "testdata.FloatingPoint") - proto.RegisterType((*MessageWithMap)(nil), "testdata.MessageWithMap") - proto.RegisterType((*Oneof)(nil), "testdata.Oneof") - proto.RegisterType((*Oneof_F_Group)(nil), "testdata.Oneof.F_Group") - proto.RegisterType((*Communique)(nil), "testdata.Communique") - proto.RegisterEnum("testdata.FOO", FOO_name, FOO_value) - proto.RegisterEnum("testdata.GoTest_KIND", GoTest_KIND_name, GoTest_KIND_value) - proto.RegisterEnum("testdata.MyMessage_Color", MyMessage_Color_name, MyMessage_Color_value) - proto.RegisterEnum("testdata.DefaultsMessage_DefaultsEnum", DefaultsMessage_DefaultsEnum_name, DefaultsMessage_DefaultsEnum_value) - proto.RegisterEnum("testdata.Defaults_Color", Defaults_Color_name, Defaults_Color_value) - proto.RegisterEnum("testdata.RepeatedEnum_Color", RepeatedEnum_Color_name, RepeatedEnum_Color_value) - proto.RegisterExtension(E_Ext_More) - proto.RegisterExtension(E_Ext_Text) - proto.RegisterExtension(E_Ext_Number) - proto.RegisterExtension(E_Greeting) - proto.RegisterExtension(E_Complex) - proto.RegisterExtension(E_RComplex) - proto.RegisterExtension(E_NoDefaultDouble) - proto.RegisterExtension(E_NoDefaultFloat) - proto.RegisterExtension(E_NoDefaultInt32) - proto.RegisterExtension(E_NoDefaultInt64) - proto.RegisterExtension(E_NoDefaultUint32) - proto.RegisterExtension(E_NoDefaultUint64) - proto.RegisterExtension(E_NoDefaultSint32) - proto.RegisterExtension(E_NoDefaultSint64) - proto.RegisterExtension(E_NoDefaultFixed32) - proto.RegisterExtension(E_NoDefaultFixed64) - proto.RegisterExtension(E_NoDefaultSfixed32) - proto.RegisterExtension(E_NoDefaultSfixed64) - proto.RegisterExtension(E_NoDefaultBool) - proto.RegisterExtension(E_NoDefaultString) - proto.RegisterExtension(E_NoDefaultBytes) - proto.RegisterExtension(E_NoDefaultEnum) - proto.RegisterExtension(E_DefaultDouble) - proto.RegisterExtension(E_DefaultFloat) - proto.RegisterExtension(E_DefaultInt32) - proto.RegisterExtension(E_DefaultInt64) - proto.RegisterExtension(E_DefaultUint32) - proto.RegisterExtension(E_DefaultUint64) - proto.RegisterExtension(E_DefaultSint32) - proto.RegisterExtension(E_DefaultSint64) - proto.RegisterExtension(E_DefaultFixed32) - proto.RegisterExtension(E_DefaultFixed64) - proto.RegisterExtension(E_DefaultSfixed32) - proto.RegisterExtension(E_DefaultSfixed64) - proto.RegisterExtension(E_DefaultBool) - proto.RegisterExtension(E_DefaultString) - proto.RegisterExtension(E_DefaultBytes) - proto.RegisterExtension(E_DefaultEnum) - proto.RegisterExtension(E_X201) - proto.RegisterExtension(E_X202) - proto.RegisterExtension(E_X203) - proto.RegisterExtension(E_X204) - proto.RegisterExtension(E_X205) - proto.RegisterExtension(E_X206) - proto.RegisterExtension(E_X207) - proto.RegisterExtension(E_X208) - proto.RegisterExtension(E_X209) - proto.RegisterExtension(E_X210) - proto.RegisterExtension(E_X211) - proto.RegisterExtension(E_X212) - proto.RegisterExtension(E_X213) - proto.RegisterExtension(E_X214) - proto.RegisterExtension(E_X215) - proto.RegisterExtension(E_X216) - proto.RegisterExtension(E_X217) - proto.RegisterExtension(E_X218) - proto.RegisterExtension(E_X219) - proto.RegisterExtension(E_X220) - proto.RegisterExtension(E_X221) - proto.RegisterExtension(E_X222) - proto.RegisterExtension(E_X223) - proto.RegisterExtension(E_X224) - proto.RegisterExtension(E_X225) - proto.RegisterExtension(E_X226) - proto.RegisterExtension(E_X227) - proto.RegisterExtension(E_X228) - proto.RegisterExtension(E_X229) - proto.RegisterExtension(E_X230) - proto.RegisterExtension(E_X231) - proto.RegisterExtension(E_X232) - proto.RegisterExtension(E_X233) - proto.RegisterExtension(E_X234) - proto.RegisterExtension(E_X235) - proto.RegisterExtension(E_X236) - proto.RegisterExtension(E_X237) - proto.RegisterExtension(E_X238) - proto.RegisterExtension(E_X239) - proto.RegisterExtension(E_X240) - proto.RegisterExtension(E_X241) - proto.RegisterExtension(E_X242) - proto.RegisterExtension(E_X243) - proto.RegisterExtension(E_X244) - proto.RegisterExtension(E_X245) - proto.RegisterExtension(E_X246) - proto.RegisterExtension(E_X247) - proto.RegisterExtension(E_X248) - proto.RegisterExtension(E_X249) - proto.RegisterExtension(E_X250) -} - -var fileDescriptor0 = []byte{ - // 4253 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x7a, 0xd9, 0x77, 0x1b, 0x47, - 0x76, 0xbe, 0x1b, 0x3b, 0x0a, 0x20, 0xd1, 0x6c, 0xc9, 0x12, 0x44, 0x2d, 0x96, 0x31, 0x63, 0x5b, - 0x92, 0x6d, 0x0e, 0xd1, 0x00, 0x49, 0x09, 0x9e, 0xdf, 0x9c, 0x23, 0x4a, 0x24, 0x87, 0x67, 0x44, - 0x82, 0xbf, 0x26, 0xed, 0x39, 0xe3, 0x3c, 0xe0, 0x80, 0x62, 0x83, 0x84, 0x05, 0xa0, 0x21, 0x00, - 0x8c, 0xa5, 0xe4, 0x25, 0x2f, 0xc9, 0x6b, 0xb6, 0x97, 0xbc, 0xe6, 0x29, 0x4f, 0x49, 0xce, 0xc9, - 0x5f, 0x11, 0xdb, 0xb3, 0x7a, 0xd6, 0xac, 0x93, 0x7d, 0x99, 0xec, 0xdb, 0x4c, 0x92, 0x93, 0x73, - 0x92, 0x7b, 0x6f, 0xdd, 0xee, 0xae, 0x6e, 0xa0, 0x9b, 0x94, 0x1e, 0x4c, 0x74, 0xd5, 0xf7, 0xdd, - 0xba, 0x55, 0xfd, 0xd5, 0xbd, 0x75, 0xab, 0x2d, 0xc4, 0xc4, 0x1e, 0x4f, 0x96, 0x86, 0x23, 0x67, - 0xe2, 0x18, 0x39, 0xfc, 0x7d, 0xd4, 0x9e, 0xb4, 0x2b, 0xb7, 0x45, 0x66, 0xcb, 0xd9, 0x18, 0x9c, - 0xf6, 0x8d, 0x57, 0x44, 0xb2, 0xe3, 0x38, 0x65, 0xed, 0x66, 0xe2, 0xd6, 0xbc, 0x39, 0xb7, 0xe4, - 0x22, 0x96, 0x36, 0x9b, 0x4d, 0x0b, 0x7b, 0x2a, 0x6b, 0xa2, 0xb0, 0xe5, 0x1c, 0x40, 0xf3, 0x66, - 0xd7, 0xee, 0x1d, 0x19, 0x17, 0x45, 0xfa, 0x51, 0xfb, 0xd0, 0xee, 0x11, 0x23, 0x6f, 0xa5, 0x7b, - 0xf8, 0x60, 0x18, 0x22, 0x75, 0xf0, 0x7c, 0x68, 0x97, 0x13, 0xd4, 0x98, 0x9a, 0xc0, 0xef, 0xca, - 0x2f, 0xde, 0xc0, 0x41, 0x90, 0x69, 0xdc, 0x16, 0xa9, 0x2f, 0x75, 0x07, 0x47, 0x3c, 0xca, 0xcb, - 0xfe, 0x28, 0xb2, 0x7f, 0xe9, 0x4b, 0xdb, 0xbb, 0x0f, 0xad, 0xd4, 0x13, 0x80, 0xa0, 0xfd, 0x83, - 0xf6, 0x61, 0x0f, 0x4d, 0x69, 0x68, 0x7f, 0x82, 0x0f, 0xd8, 0xba, 0xd7, 0x1e, 0xb5, 0xfb, 0xe5, - 0x24, 0xb4, 0xa6, 0xad, 0xf4, 0x10, 0x1f, 0x8c, 0x77, 0xc4, 0x9c, 0x65, 0x3f, 0x3d, 0xed, 0x8e, - 0xec, 0x23, 0x72, 0xae, 0x9c, 0x02, 0xfb, 0x85, 0x69, 0xfb, 0xd4, 0x69, 0xcd, 0x8d, 0x54, 0xac, - 0x24, 0x0f, 0xed, 0xf6, 0xc4, 0x25, 0xa7, 0x6f, 0x26, 0x63, 0xc9, 0x0a, 0x16, 0xc9, 0xcd, 0xe1, - 0xa4, 0xeb, 0x0c, 0xda, 0x3d, 0x49, 0xce, 0x80, 0x5f, 0xd1, 0x64, 0x47, 0xc5, 0x1a, 0xaf, 0x8b, - 0xd2, 0x66, 0x6b, 0xdd, 0x71, 0x7a, 0x2d, 0xd7, 0xa3, 0xb2, 0x00, 0xc7, 0x73, 0xd6, 0x5c, 0x07, - 0x5b, 0xdd, 0x29, 0x19, 0xb7, 0x84, 0xbe, 0xd9, 0xda, 0x1e, 0x4c, 0x6a, 0xa6, 0x0f, 0x2c, 0x00, - 0x30, 0x6d, 0xcd, 0x77, 0xa8, 0x79, 0x0a, 0xb9, 0x5a, 0xf7, 0x91, 0x45, 0x40, 0x26, 0x25, 0x72, - 0xb5, 0xee, 0x21, 0xdf, 0x12, 0xc6, 0x66, 0x6b, 0xb3, 0xfb, 0xcc, 0x3e, 0x52, 0xad, 0xce, 0x01, - 0x36, 0x6b, 0xe9, 0x1d, 0xee, 0x98, 0x81, 0x56, 0x2d, 0xcf, 0x03, 0x3a, 0xe3, 0xa2, 0x15, 0xdb, - 0x77, 0xc4, 0xc2, 0x66, 0xeb, 0xdd, 0x6e, 0xd0, 0xe1, 0x12, 0x80, 0xe7, 0xac, 0x52, 0x47, 0xb6, - 0x4f, 0x63, 0x55, 0xc3, 0x3a, 0x60, 0x53, 0x8c, 0x55, 0xec, 0xd2, 0xec, 0x36, 0x7b, 0x4e, 0x7b, - 0xe2, 0x43, 0x17, 0x00, 0x9a, 0x80, 0xd9, 0x51, 0x73, 0xd0, 0xea, 0x43, 0xe7, 0x14, 0x24, 0xe3, - 0x43, 0x0d, 0x80, 0x6a, 0x60, 0x55, 0xb6, 0x07, 0xb1, 0xfb, 0x93, 0x51, 0x77, 0x70, 0xec, 0x63, - 0x2f, 0x90, 0x7e, 0x4b, 0x1d, 0xd9, 0x1e, 0xf4, 0x60, 0xfd, 0x39, 0xbc, 0x5c, 0x1f, 0x6a, 0x03, - 0xb4, 0x08, 0x1e, 0x50, 0x73, 0xc8, 0x6a, 0x68, 0x0d, 0x3a, 0x00, 0x5d, 0x40, 0xab, 0x33, 0xd6, - 0x60, 0x3f, 0xb4, 0x06, 0xc7, 0x80, 0x35, 0x18, 0xab, 0xac, 0x81, 0xaa, 0x19, 0x29, 0xc4, 0xf2, - 0x45, 0xd0, 0xab, 0xaf, 0x19, 0xd9, 0x18, 0xd4, 0x0c, 0x03, 0x5f, 0x06, 0xa0, 0xa2, 0x99, 0x10, - 0x92, 0x06, 0x67, 0xe4, 0x25, 0x40, 0x2a, 0x9a, 0x61, 0x64, 0x48, 0x33, 0x8c, 0xbd, 0x0c, 0xd8, - 0x80, 0x66, 0xa6, 0xd0, 0xaa, 0xe5, 0x32, 0xa0, 0x03, 0x9a, 0x61, 0x74, 0x50, 0x33, 0x0c, 0xbe, - 0x02, 0x60, 0x55, 0x33, 0x61, 0xac, 0x6a, 0x78, 0x11, 0xb0, 0xaa, 0x66, 0xd4, 0xd9, 0xb9, 0x9a, - 0x61, 0xe8, 0x55, 0x80, 0x2a, 0x9a, 0x51, 0xad, 0x7a, 0x9a, 0x61, 0xe8, 0x35, 0x80, 0xaa, 0x9a, - 0x51, 0xb1, 0x9e, 0x66, 0x18, 0x7b, 0x1d, 0xb0, 0xaa, 0x66, 0x18, 0x7b, 0x5b, 0xd5, 0x0c, 0x43, - 0x3f, 0xd2, 0x00, 0xab, 0x88, 0x86, 0xa1, 0x6f, 0x06, 0x44, 0xc3, 0xd8, 0x8f, 0x11, 0xab, 0xaa, - 0x26, 0x0c, 0x56, 0x57, 0xe1, 0x13, 0x04, 0xab, 0xb2, 0x61, 0xb0, 0x2f, 0x1b, 0x37, 0x04, 0x95, - 0x6f, 0x40, 0xa4, 0x72, 0x65, 0xe3, 0xc6, 0x30, 0x55, 0x36, 0x1e, 0xf0, 0x15, 0x0a, 0xb5, 0x2c, - 0x9b, 0x29, 0x24, 0x8c, 0xee, 0x21, 0x6f, 0x02, 0xd2, 0x93, 0x8d, 0x87, 0x0c, 0xc8, 0xc6, 0xc3, - 0xbe, 0x0a, 0x58, 0x45, 0x36, 0x33, 0xd0, 0xaa, 0xe5, 0x0a, 0xa0, 0x15, 0xd9, 0x78, 0x68, 0x55, - 0x36, 0x1e, 0xf8, 0x33, 0x00, 0xf6, 0x65, 0x33, 0x8d, 0x55, 0x0d, 0x7f, 0x16, 0xb0, 0xbe, 0x6c, - 0x82, 0xb3, 0x93, 0xb2, 0xf1, 0xa0, 0xaf, 0x01, 0xd4, 0x93, 0x4d, 0xd0, 0x2a, 0xcb, 0xc6, 0x83, - 0xbe, 0x0e, 0x50, 0x5f, 0x36, 0x41, 0x2c, 0xcb, 0xc6, 0xc3, 0xbe, 0x41, 0xf9, 0xcd, 0x95, 0x8d, - 0x87, 0x55, 0x64, 0xe3, 0x41, 0x7f, 0x13, 0x73, 0xa1, 0x27, 0x1b, 0x0f, 0xaa, 0xca, 0xc6, 0xc3, - 0xfe, 0x16, 0x62, 0x7d, 0xd9, 0x4c, 0x83, 0xd5, 0x55, 0xf8, 0x6d, 0x04, 0xfb, 0xb2, 0xf1, 0xc0, - 0x4b, 0xe4, 0x04, 0xca, 0xe6, 0xc8, 0xee, 0xb4, 0x4f, 0x7b, 0x28, 0xb1, 0x5b, 0xa8, 0x9b, 0x46, - 0x6a, 0x32, 0x3a, 0xb5, 0xd1, 0x13, 0xe8, 0x7c, 0xe8, 0xf6, 0x01, 0x7e, 0xc1, 0x95, 0x8f, 0x4f, - 0xb8, 0x8d, 0xfa, 0x69, 0x24, 0x40, 0xba, 0x25, 0xa9, 0xa1, 0x69, 0x3c, 0xf8, 0xe2, 0xe3, 0xef, - 0xa0, 0x8a, 0x1a, 0x09, 0x50, 0x6f, 0x49, 0x2a, 0xc9, 0xc7, 0xd7, 0xc4, 0x05, 0x5f, 0x4a, 0x3e, - 0xe3, 0x4d, 0xd4, 0x52, 0x23, 0x59, 0x33, 0x97, 0xad, 0x05, 0x57, 0x50, 0xb3, 0x48, 0x81, 0x61, - 0xde, 0x42, 0x49, 0x35, 0x92, 0xab, 0x75, 0x8f, 0xa4, 0x8e, 0x64, 0xa2, 0x0c, 0x59, 0x58, 0x3e, - 0xe7, 0x6d, 0x54, 0x56, 0x23, 0x05, 0x03, 0x2d, 0x83, 0x18, 0x65, 0xf7, 0x0c, 0x4e, 0x60, 0x9c, - 0x25, 0x54, 0x58, 0x23, 0x05, 0xe3, 0xb8, 0x9c, 0xe0, 0x38, 0x0b, 0xae, 0xd0, 0x7c, 0xca, 0xe7, - 0x50, 0x69, 0x8d, 0x4c, 0xad, 0x5a, 0xaf, 0xae, 0xdc, 0x83, 0x55, 0xa0, 0x7e, 0x9f, 0x53, 0xc7, - 0x71, 0x58, 0x72, 0x3e, 0x69, 0x19, 0x35, 0xd7, 0xc8, 0x98, 0x6b, 0xd5, 0xbb, 0xe6, 0x5d, 0x18, - 0x49, 0x02, 0x7c, 0xd6, 0x17, 0x90, 0xc5, 0xe2, 0xf3, 0x59, 0x55, 0x54, 0x5f, 0x43, 0x3f, 0xb1, - 0x7b, 0x3d, 0xe7, 0xad, 0x9b, 0x95, 0x0f, 0x9d, 0x51, 0xef, 0xe8, 0xd5, 0x8a, 0x00, 0xbe, 0x84, - 0xaa, 0xa3, 0x2e, 0xb8, 0x82, 0xf4, 0xe9, 0xbf, 0x8c, 0xe7, 0xb0, 0x62, 0x23, 0xbb, 0xde, 0x3d, - 0x1e, 0x38, 0x63, 0x1b, 0x7c, 0x25, 0x44, 0x68, 0x4d, 0xf6, 0xc3, 0xeb, 0xf8, 0x2b, 0x48, 0x5b, - 0x68, 0x24, 0xdf, 0x06, 0x51, 0xe8, 0xac, 0xd0, 0x19, 0x9c, 0xc0, 0x3a, 0xfe, 0x2a, 0x72, 0x0c, - 0xe0, 0x80, 0x30, 0x74, 0x16, 0xaa, 0xcf, 0x59, 0x13, 0x97, 0x42, 0x79, 0xb1, 0x35, 0x6c, 0x3f, - 0x7e, 0x02, 0x3c, 0x13, 0xd3, 0xe3, 0x7a, 0x42, 0xd7, 0xac, 0x0b, 0x81, 0x14, 0xb9, 0x47, 0xdd, - 0xc6, 0x3d, 0x71, 0x39, 0x9c, 0x28, 0x5d, 0x66, 0x0d, 0xf3, 0x25, 0x31, 0x2f, 0x06, 0x73, 0x66, - 0x88, 0xaa, 0x04, 0x60, 0x97, 0x5a, 0xc7, 0x04, 0xea, 0x53, 0xfd, 0x48, 0xcc, 0xd4, 0xff, 0x27, - 0xae, 0x4c, 0xa7, 0x52, 0x97, 0xbc, 0x82, 0x19, 0x95, 0xc8, 0x97, 0xc2, 0x59, 0x75, 0x8a, 0x3e, - 0x63, 0xec, 0x55, 0x4c, 0xb1, 0x2a, 0x7d, 0x6a, 0xf4, 0x77, 0x44, 0x79, 0x2a, 0xd9, 0xba, 0xec, - 0x35, 0xcc, 0xb9, 0xc4, 0x7e, 0x39, 0x94, 0x77, 0xc3, 0xe4, 0x19, 0x43, 0xdf, 0xc5, 0x24, 0xac, - 0x90, 0xa7, 0x46, 0xa6, 0x25, 0x0b, 0xa6, 0x63, 0x97, 0x7b, 0x0f, 0xb3, 0x32, 0x2f, 0x59, 0x20, - 0x33, 0xab, 0xe3, 0x86, 0xf2, 0xb3, 0xcb, 0x6d, 0x60, 0x9a, 0xe6, 0x71, 0x83, 0xa9, 0x9a, 0xc9, - 0x9f, 0x47, 0xf2, 0xfe, 0xec, 0x19, 0xff, 0x38, 0x89, 0x09, 0x96, 0xd9, 0xfb, 0xb3, 0xa6, 0xec, - 0xb1, 0x67, 0x4c, 0xf9, 0x27, 0xc8, 0x36, 0x14, 0xf6, 0xd4, 0x9c, 0x1f, 0x0a, 0xaf, 0xe2, 0x38, - 0x1e, 0x39, 0xa7, 0xc3, 0xf2, 0x26, 0x1c, 0xed, 0x84, 0x79, 0x63, 0xaa, 0xfa, 0x71, 0x0f, 0x79, - 0x5b, 0x88, 0xb2, 0x82, 0x24, 0x69, 0x45, 0xda, 0x95, 0x56, 0xf6, 0x60, 0xdc, 0xd9, 0x56, 0x24, - 0xca, 0xb3, 0xa2, 0x90, 0xd0, 0x8a, 0x1b, 0xf4, 0xa5, 0x95, 0xf7, 0x61, 0x53, 0xcd, 0xb2, 0xe2, - 0xa6, 0x00, 0xb6, 0x12, 0x20, 0x2d, 0xae, 0xf8, 0xf5, 0x16, 0xf5, 0x1b, 0x9f, 0x0d, 0x17, 0x60, - 0x5b, 0x74, 0x7e, 0x0e, 0x56, 0x5a, 0x92, 0xa6, 0x38, 0x37, 0x4d, 0xfb, 0xff, 0x11, 0xb4, 0x80, - 0x37, 0xd3, 0xb4, 0x9f, 0x9a, 0x41, 0xab, 0xfc, 0x9a, 0x06, 0xc5, 0x26, 0xd4, 0x93, 0x46, 0x4e, - 0xa4, 0xde, 0x6b, 0x6e, 0x3f, 0xd4, 0x5f, 0xc2, 0x5f, 0xeb, 0xcd, 0xe6, 0x23, 0x5d, 0x33, 0xf2, - 0x22, 0xbd, 0xfe, 0x95, 0x83, 0x8d, 0x7d, 0x3d, 0x61, 0x94, 0x44, 0x61, 0x73, 0x7b, 0x77, 0x6b, - 0xc3, 0xda, 0xb3, 0xb6, 0x77, 0x0f, 0xf4, 0x24, 0xf6, 0x6d, 0x3e, 0x6a, 0xde, 0x3f, 0xd0, 0x53, - 0x46, 0x56, 0x24, 0xb1, 0x2d, 0x6d, 0x08, 0x91, 0xd9, 0x3f, 0x80, 0xfe, 0x2d, 0x3d, 0x83, 0x56, - 0x0e, 0xb6, 0x77, 0x36, 0xf4, 0x2c, 0x22, 0x0f, 0xde, 0xdd, 0x7b, 0xb4, 0xa1, 0xe7, 0xf0, 0xe7, - 0x7d, 0xcb, 0xba, 0xff, 0x15, 0x3d, 0x8f, 0xa4, 0x9d, 0xfb, 0x7b, 0xba, 0xa0, 0xee, 0xfb, 0xeb, - 0xd0, 0x5d, 0x30, 0x8a, 0x22, 0xb7, 0xf9, 0xee, 0xee, 0x83, 0x83, 0xed, 0xe6, 0xae, 0x5e, 0xac, - 0xfc, 0x7a, 0x42, 0x88, 0x2d, 0x67, 0xff, 0x49, 0x77, 0x48, 0x55, 0xf1, 0x75, 0x21, 0xc6, 0xf0, - 0xbb, 0x45, 0xd2, 0xe3, 0xca, 0x2e, 0x8f, 0x2d, 0x14, 0x74, 0x8c, 0x57, 0x45, 0x91, 0xba, 0x3b, - 0x32, 0x14, 0x50, 0x41, 0x97, 0xb5, 0x0a, 0xd8, 0xc6, 0xd1, 0x21, 0x08, 0x59, 0xad, 0x53, 0x1d, - 0x97, 0x51, 0x20, 0xab, 0x75, 0xa8, 0xef, 0xe9, 0xb1, 0x35, 0xa6, 0xb0, 0x4e, 0xb5, 0x5b, 0xde, - 0xa2, 0x71, 0x65, 0xa0, 0x07, 0x91, 0xd3, 0x98, 0x52, 0x16, 0xa5, 0x69, 0x89, 0xba, 0xee, 0x2e, - 0xe1, 0x0f, 0x29, 0x0b, 0x9f, 0xb0, 0xd8, 0x14, 0x79, 0xaf, 0x1d, 0xc7, 0xa2, 0x56, 0x9e, 0x91, - 0x4e, 0x33, 0x12, 0xd4, 0xe4, 0x4d, 0x49, 0x02, 0xd8, 0x9b, 0x05, 0xf2, 0x46, 0x92, 0xa4, 0x3b, - 0x95, 0xeb, 0x62, 0x6e, 0xd7, 0x19, 0xc8, 0x2d, 0x44, 0xab, 0x54, 0x14, 0x5a, 0xbb, 0xac, 0x51, - 0x09, 0xa3, 0xb5, 0x2b, 0x37, 0x84, 0x50, 0xfa, 0x74, 0xa1, 0x1d, 0xca, 0x3e, 0xda, 0x88, 0xda, - 0x61, 0xe5, 0x4d, 0x91, 0xd9, 0x69, 0x3f, 0x3b, 0x68, 0x1f, 0xc3, 0x58, 0xa2, 0xd7, 0x1e, 0x4f, - 0x60, 0x6d, 0x50, 0x2a, 0xff, 0x0b, 0xff, 0x34, 0x3a, 0x71, 0xe5, 0xb1, 0x55, 0x4a, 0xe5, 0xa9, - 0x10, 0xcd, 0xde, 0xd1, 0x8e, 0x3d, 0x1e, 0xb7, 0x8f, 0x6d, 0x38, 0x2f, 0x64, 0x06, 0x60, 0xd4, - 0xc6, 0x6b, 0x0a, 0x2c, 0xe6, 0xaf, 0xfa, 0xab, 0xe0, 0xa3, 0x96, 0x76, 0x09, 0x62, 0x31, 0x14, - 0x3c, 0x48, 0x0e, 0x4e, 0xfb, 0x74, 0x59, 0x91, 0xb6, 0xf0, 0xe7, 0xe2, 0x35, 0x91, 0x91, 0x18, - 0xbc, 0x14, 0x19, 0xb4, 0xfb, 0x76, 0x59, 0x8e, 0x4b, 0xbf, 0x2b, 0xbf, 0xa4, 0x09, 0xb1, 0x6b, - 0x7f, 0x78, 0x8e, 0x31, 0x7d, 0x54, 0xcc, 0x98, 0x49, 0x39, 0xe6, 0x3b, 0x71, 0x63, 0xa2, 0xce, - 0x3a, 0x8e, 0x73, 0xd4, 0x92, 0xaf, 0x58, 0xde, 0xab, 0xe4, 0xb1, 0x85, 0xde, 0x5a, 0xe5, 0x7d, - 0x51, 0xdc, 0x1e, 0x0c, 0xec, 0x91, 0xeb, 0x13, 0x98, 0x38, 0x71, 0xc6, 0x13, 0xbe, 0xe0, 0xa1, - 0xdf, 0x46, 0x59, 0xa4, 0x86, 0xce, 0x68, 0x22, 0xe7, 0xd9, 0x48, 0xc1, 0x99, 0x66, 0xd9, 0xa2, - 0x16, 0xe3, 0x9a, 0xc8, 0x3f, 0x76, 0x80, 0xfe, 0x18, 0x27, 0x91, 0xa4, 0xda, 0xc2, 0x6f, 0xa8, - 0xfc, 0x82, 0x26, 0x8a, 0xcd, 0xc9, 0x89, 0x6f, 0x1c, 0x7c, 0x7f, 0x62, 0x3f, 0x27, 0xf7, 0xc0, - 0x77, 0xf8, 0x89, 0x57, 0x3b, 0x3f, 0xdd, 0xee, 0x9d, 0xca, 0x0b, 0x9f, 0xa2, 0x25, 0x1f, 0x8c, - 0x4b, 0x22, 0xf3, 0xa1, 0xdd, 0x3d, 0x3e, 0x99, 0x90, 0xcd, 0x84, 0xc5, 0x4f, 0x50, 0x26, 0xa4, - 0xbb, 0xe8, 0x6c, 0x39, 0x45, 0xeb, 0x75, 0xc9, 0x5f, 0x2f, 0x75, 0x0e, 0x96, 0x04, 0xdd, 0xc9, - 0xe5, 0x8e, 0xf4, 0x9f, 0x83, 0x7f, 0x89, 0xca, 0xff, 0x24, 0x45, 0x7e, 0xe7, 0xb9, 0xeb, 0x05, - 0x8c, 0xf9, 0xd8, 0x39, 0x1d, 0xc8, 0x39, 0xa6, 0x2d, 0xf9, 0xe0, 0xad, 0x5d, 0x42, 0x59, 0x3b, - 0x40, 0x3e, 0x3d, 0x75, 0x26, 0x36, 0xb9, 0x91, 0xb7, 0xe4, 0x03, 0xce, 0x62, 0x68, 0x4f, 0xc0, - 0x07, 0xac, 0xfc, 0xf0, 0xa7, 0xef, 0x57, 0xfa, 0x1c, 0x7e, 0xc1, 0xf9, 0x37, 0xe3, 0xe0, 0xaa, - 0x8c, 0xcb, 0x19, 0xba, 0x74, 0x52, 0xe0, 0xea, 0x6a, 0x59, 0x8c, 0x02, 0x99, 0xe4, 0x21, 0x9e, - 0xb7, 0xe4, 0x08, 0xc5, 0x30, 0x25, 0x30, 0x42, 0x0e, 0x80, 0xd4, 0x60, 0xac, 0x88, 0xdc, 0x61, - 0xf7, 0x89, 0x3d, 0x3e, 0x81, 0x17, 0x93, 0x05, 0xaf, 0xe6, 0xcd, 0x2b, 0x3e, 0xc7, 0x5b, 0x8b, - 0xa5, 0x07, 0x4e, 0xcf, 0x19, 0x59, 0x1e, 0x14, 0xf2, 0x6d, 0x7e, 0xec, 0xf4, 0x6d, 0x29, 0x96, - 0x1c, 0xa5, 0x89, 0xeb, 0xb3, 0x78, 0xfb, 0x00, 0x72, 0xc3, 0x81, 0x8b, 0x37, 0xae, 0x4a, 0x47, - 0x0f, 0xf1, 0x30, 0x58, 0x16, 0x54, 0xec, 0xa2, 0x43, 0x74, 0x38, 0x34, 0x16, 0xd1, 0xa1, 0xe3, - 0x0e, 0xe6, 0x78, 0x88, 0x76, 0x58, 0x29, 0x79, 0xcf, 0x8b, 0x6f, 0x41, 0x1c, 0x71, 0x0d, 0xfa, - 0x71, 0x44, 0xee, 0xdd, 0x3c, 0x6d, 0x2e, 0x19, 0x47, 0xe4, 0xc6, 0x7d, 0x4d, 0xa4, 0xc9, 0x6d, - 0x8c, 0xb9, 0xd6, 0x06, 0x86, 0x78, 0x88, 0xb9, 0x5b, 0xd6, 0xc6, 0xc6, 0x2e, 0xc4, 0x78, 0x8c, - 0xf6, 0x8f, 0xde, 0xdd, 0xd0, 0x13, 0xca, 0xeb, 0xff, 0x0d, 0x4d, 0x24, 0x37, 0x9e, 0xd1, 0x2b, - 0xc6, 0x69, 0xb8, 0xdb, 0x03, 0x7f, 0x9b, 0xab, 0x22, 0xd5, 0x77, 0x46, 0xb6, 0x71, 0x61, 0xc6, - 0x2c, 0xcb, 0xc7, 0xf4, 0x3a, 0x95, 0x7b, 0x51, 0xb0, 0x62, 0x11, 0xde, 0x7c, 0x43, 0xa4, 0x26, - 0x36, 0xd8, 0x9c, 0xc9, 0x3b, 0x91, 0x03, 0x20, 0xc0, 0x84, 0x98, 0x04, 0x9b, 0xf4, 0x10, 0x5e, - 0xc9, 0x4c, 0x68, 0x97, 0xa6, 0xc7, 0x90, 0xca, 0x7b, 0x42, 0x7f, 0xe0, 0xf4, 0x87, 0x3d, 0xfb, - 0x19, 0x8c, 0x64, 0x0f, 0xc6, 0x90, 0xff, 0x50, 0x84, 0x9d, 0xee, 0x88, 0xb6, 0x24, 0xdd, 0x7e, - 0xd2, 0x03, 0x6e, 0x91, 0xb1, 0x0d, 0x5b, 0xed, 0x88, 0xa3, 0x0f, 0x3f, 0x21, 0x7a, 0x72, 0xd2, - 0x1d, 0xe1, 0x6e, 0xc4, 0xa0, 0x29, 0x1f, 0x2a, 0x5b, 0xa2, 0xc4, 0xa7, 0xe6, 0x31, 0x0f, 0x5c, - 0xb9, 0x23, 0x8a, 0x6e, 0x13, 0x5d, 0x05, 0xc3, 0xc2, 0xbd, 0xbf, 0x61, 0x35, 0x61, 0x35, 0x61, - 0x59, 0x9b, 0xbb, 0x1b, 0xb0, 0x96, 0xf0, 0xe3, 0xe0, 0xcb, 0xcd, 0xc0, 0x52, 0x5e, 0x13, 0x45, - 0xcf, 0xf7, 0x7d, 0x7b, 0x42, 0x3d, 0x18, 0x5d, 0xb3, 0x8d, 0x44, 0x4e, 0xab, 0x64, 0x45, 0x7a, - 0xa3, 0x3f, 0x9c, 0x3c, 0xaf, 0xfc, 0xac, 0x28, 0x30, 0xe8, 0x51, 0x17, 0x9c, 0x5d, 0x13, 0xd9, - 0x3e, 0xcf, 0x57, 0xa3, 0x03, 0x8c, 0xaa, 0x29, 0x1f, 0xe7, 0xfe, 0xb6, 0x5c, 0xf4, 0x62, 0x4d, - 0x64, 0x95, 0xc0, 0xc4, 0xfb, 0x33, 0xa1, 0xee, 0x4f, 0xb9, 0x93, 0x93, 0xca, 0x4e, 0xae, 0xec, - 0x88, 0xac, 0x4c, 0x27, 0x63, 0x4a, 0x91, 0xb2, 0xf8, 0x91, 0x62, 0x92, 0x6f, 0xbe, 0x20, 0xdb, - 0xe4, 0x7d, 0x2c, 0xc8, 0x8d, 0x04, 0xcb, 0x08, 0x19, 0x87, 0x04, 0x35, 0x49, 0xb9, 0xfd, 0x4e, - 0x5a, 0xe4, 0xdc, 0x95, 0x02, 0x89, 0x67, 0x64, 0xc5, 0x41, 0xa6, 0xdc, 0x8a, 0x38, 0x4d, 0x35, - 0x06, 0x74, 0x66, 0xb9, 0xaa, 0xe0, 0x50, 0x89, 0xe5, 0x6f, 0x46, 0x56, 0x11, 0x5e, 0x27, 0x24, - 0xea, 0xa4, 0x57, 0xeb, 0x66, 0x64, 0x9d, 0x60, 0xdc, 0x14, 0x79, 0xaf, 0x32, 0xa0, 0xe0, 0xc6, - 0x85, 0x6d, 0xce, 0x2d, 0x05, 0x14, 0x04, 0x18, 0x48, 0xfb, 0x55, 0x6c, 0xae, 0xe3, 0xe7, 0xfa, - 0x9c, 0x7b, 0xbe, 0xa7, 0x0b, 0x69, 0xb7, 0x64, 0xcd, 0xf2, 0x89, 0xde, 0x07, 0x80, 0x85, 0xac, - 0x52, 0x9f, 0x66, 0xf9, 0xd4, 0x0e, 0x80, 0x2c, 0x9f, 0xd3, 0x69, 0xeb, 0xfb, 0xc5, 0x68, 0x46, - 0x9e, 0xcd, 0x61, 0x39, 0x73, 0xee, 0x69, 0x9c, 0xf6, 0xa5, 0x5f, 0x79, 0x66, 0xf9, 0x04, 0x6e, - 0xbc, 0x89, 0x10, 0xb9, 0xfc, 0x10, 0x02, 0x66, 0x97, 0x99, 0x59, 0x2e, 0x33, 0x61, 0x52, 0x59, - 0xae, 0x2e, 0x29, 0x24, 0x28, 0x25, 0x65, 0x46, 0x96, 0x94, 0xc6, 0x0d, 0x32, 0x27, 0x27, 0x55, - 0xf4, 0xcb, 0xc7, 0x2c, 0x1f, 0xd9, 0xfd, 0x7e, 0x3a, 0xff, 0x78, 0xa5, 0x62, 0x96, 0x0f, 0xe5, - 0xc6, 0x2a, 0xbe, 0x2f, 0xd4, 0x37, 0x9c, 0x7d, 0x30, 0x08, 0x96, 0x7d, 0xe1, 0xb9, 0xef, 0x54, - 0xc6, 0xc0, 0x86, 0x8c, 0x20, 0xf0, 0x2a, 0x69, 0x37, 0x2c, 0x22, 0x6f, 0xaf, 0x3b, 0xe8, 0xc0, - 0xa1, 0x08, 0x57, 0x22, 0x09, 0x3f, 0xa1, 0x0f, 0x5b, 0xa4, 0x06, 0x76, 0xb1, 0x4f, 0xa7, 0xbe, - 0xd4, 0xdb, 0xb2, 0x13, 0x9b, 0x20, 0x57, 0xa6, 0xa1, 0xb3, 0x3d, 0x80, 0xd3, 0x0d, 0xf1, 0x06, - 0xed, 0x81, 0x95, 0xea, 0x40, 0x83, 0xf1, 0x86, 0x48, 0x8e, 0x4f, 0x0f, 0xcb, 0x46, 0xf8, 0x5b, - 0xc1, 0xfe, 0xe9, 0xa1, 0xeb, 0x8a, 0x85, 0x08, 0xb0, 0x9f, 0x03, 0x81, 0xb6, 0x7e, 0xc6, 0x1e, - 0x39, 0xe5, 0x0b, 0xb4, 0x84, 0x2f, 0x59, 0x59, 0x68, 0x79, 0x1f, 0x1a, 0xce, 0x19, 0xfc, 0xe0, - 0xa4, 0x54, 0x50, 0xec, 0xc2, 0x71, 0x57, 0x1b, 0xc8, 0xb4, 0xdb, 0xd0, 0xd6, 0x2c, 0x6d, 0x50, - 0x39, 0x10, 0x45, 0xf7, 0x54, 0x4e, 0xf3, 0x35, 0x71, 0x27, 0x81, 0x59, 0xda, 0x9f, 0xf3, 0xe6, - 0x35, 0xdf, 0x3d, 0x15, 0xc6, 0xe9, 0x42, 0x42, 0x2b, 0x7a, 0xc8, 0x15, 0xad, 0xf2, 0x43, 0x48, - 0xf8, 0x3b, 0x10, 0x1d, 0xbd, 0x1b, 0x48, 0xd8, 0xa0, 0x87, 0xb0, 0x33, 0xc6, 0x64, 0x36, 0x67, - 0xc9, 0x07, 0xe3, 0x35, 0x51, 0xa4, 0x1f, 0x6e, 0x35, 0x95, 0xf0, 0x8a, 0xf5, 0x02, 0xb5, 0x73, - 0x09, 0x05, 0x3b, 0x1e, 0x5e, 0xe2, 0x98, 0x23, 0x19, 0xfd, 0x36, 0x3e, 0x23, 0x0a, 0xf8, 0xd7, - 0x65, 0xa6, 0xbc, 0xd3, 0x9f, 0xc0, 0x66, 0x26, 0xbe, 0x21, 0xe6, 0xe8, 0xed, 0x7b, 0xb0, 0xac, - 0x57, 0x98, 0x17, 0x65, 0x07, 0x03, 0xcb, 0x22, 0x2b, 0x43, 0xc1, 0x98, 0xbe, 0xff, 0xe4, 0x2d, - 0xf7, 0x11, 0xc3, 0x2b, 0x1d, 0xab, 0x65, 0x8e, 0xce, 0x5a, 0xfc, 0x54, 0xb9, 0x2f, 0x72, 0x94, - 0xa5, 0xe0, 0x4c, 0x68, 0x54, 0x84, 0x76, 0x5c, 0xb6, 0x29, 0x47, 0x5e, 0x54, 0xce, 0xcc, 0xdc, - 0xbd, 0xb4, 0x65, 0x69, 0xc7, 0x8b, 0x0b, 0x42, 0xdb, 0xc2, 0x43, 0xec, 0x33, 0x0e, 0xd3, 0xda, - 0xb3, 0x4a, 0x93, 0x4d, 0xc0, 0x11, 0x2f, 0xce, 0x04, 0x74, 0x4b, 0x13, 0xaf, 0x4c, 0x99, 0xc0, - 0xa7, 0xe7, 0xfc, 0x31, 0x4c, 0x7b, 0x8e, 0x87, 0x66, 0xda, 0x9e, 0xe0, 0xf8, 0x9e, 0x03, 0xf3, - 0xc3, 0xee, 0x0e, 0x1d, 0x6e, 0xe0, 0x50, 0xdc, 0xa9, 0x7c, 0x9a, 0x12, 0xf3, 0x1c, 0x44, 0xbf, - 0xdc, 0x9d, 0x9c, 0xec, 0xb4, 0x87, 0xc6, 0x23, 0x51, 0xc4, 0xf8, 0xd9, 0xea, 0xb7, 0x87, 0x43, - 0xdc, 0xa8, 0x1a, 0x1d, 0x2a, 0x6e, 0x4f, 0x05, 0x65, 0xc6, 0x2f, 0xed, 0x02, 0x78, 0x47, 0x62, - 0x37, 0x06, 0x93, 0xd1, 0x73, 0xab, 0x30, 0xf0, 0x5b, 0x8c, 0x6d, 0x51, 0xe8, 0x8f, 0x8f, 0x3d, - 0x63, 0x09, 0x32, 0x76, 0x2b, 0xd2, 0xd8, 0xce, 0xf8, 0x38, 0x60, 0x4b, 0xf4, 0xbd, 0x06, 0x74, - 0x0c, 0x23, 0xaf, 0x67, 0x2b, 0x79, 0x86, 0x63, 0x18, 0x24, 0x82, 0x8e, 0x1d, 0xfa, 0x2d, 0x50, - 0xf7, 0x0a, 0xdc, 0x48, 0x13, 0x07, 0x2b, 0x0e, 0xd2, 0x4a, 0xc1, 0x7c, 0x3d, 0xd2, 0x16, 0xc4, - 0xa4, 0x03, 0x07, 0xfe, 0x23, 0x0d, 0xe1, 0x16, 0xa4, 0xc7, 0xc5, 0x2f, 0x08, 0x3d, 0x3c, 0x7f, - 0xf5, 0x20, 0x9b, 0x9e, 0x71, 0x90, 0xcd, 0xf3, 0x41, 0xb6, 0x91, 0xb8, 0xab, 0x2d, 0xbe, 0x27, - 0x4a, 0xa1, 0x29, 0xab, 0x74, 0x43, 0xd2, 0xdf, 0x56, 0xe9, 0x05, 0xf3, 0xb2, 0xf2, 0x29, 0x56, - 0x7d, 0xb5, 0xaa, 0x5d, 0xf0, 0x2b, 0x3c, 0x7d, 0xd5, 0x70, 0x2e, 0xe6, 0x80, 0x4d, 0xfc, 0x77, - 0xc4, 0x5c, 0x60, 0xca, 0x2a, 0x39, 0x7f, 0xc6, 0xa4, 0x2a, 0x3f, 0x9f, 0x16, 0xe9, 0xe6, 0xc0, - 0x76, 0x3a, 0xc6, 0xe5, 0x60, 0x46, 0xfc, 0xe2, 0x4b, 0x6e, 0x36, 0xbc, 0x12, 0xca, 0x86, 0xd0, - 0xe3, 0xe6, 0xc2, 0x2b, 0xa1, 0x5c, 0xe8, 0x76, 0x41, 0xc0, 0xbe, 0x3e, 0x95, 0x09, 0xa1, 0xd3, - 0x4f, 0x83, 0xd7, 0xa7, 0xd2, 0xa0, 0xdf, 0x0d, 0xec, 0xab, 0xe1, 0x1c, 0x08, 0xbd, 0x5e, 0xfe, - 0xbb, 0x1a, 0xce, 0x7f, 0x5e, 0x27, 0x30, 0xaf, 0x84, 0x72, 0x1f, 0xb9, 0x24, 0xb3, 0xde, 0xd5, - 0x70, 0xd6, 0x23, 0x1e, 0xe7, 0xbb, 0xab, 0xe1, 0x7c, 0x47, 0x9d, 0x9c, 0xdf, 0xae, 0x84, 0xf2, - 0x1b, 0x19, 0x95, 0x89, 0xed, 0x6a, 0x38, 0xb1, 0x49, 0x9e, 0xe2, 0xa9, 0x9a, 0xd5, 0xbc, 0x4e, - 0xf0, 0xd4, 0x0c, 0xa5, 0xb4, 0xe8, 0x73, 0x3d, 0xbd, 0x0b, 0x0a, 0xef, 0x75, 0x5c, 0x36, 0xf7, - 0xc8, 0x59, 0x8a, 0xf9, 0x5a, 0x4d, 0xab, 0xe9, 0x1e, 0xb9, 0x4c, 0x91, 0xed, 0x70, 0xdd, 0xa8, - 0x53, 0x8c, 0x52, 0x64, 0x49, 0x2f, 0x7f, 0x69, 0xb3, 0x45, 0xb1, 0x8a, 0xe6, 0x25, 0x4f, 0xef, - 0xb7, 0x20, 0x18, 0xb5, 0x1e, 0xb5, 0x47, 0xc7, 0x00, 0x6c, 0x41, 0x25, 0xee, 0xd5, 0xde, 0xf8, - 0xfe, 0x0b, 0x1d, 0xee, 0xc1, 0x12, 0xfd, 0x92, 0x2b, 0xae, 0x23, 0xea, 0xd5, 0x58, 0x5e, 0x8b, - 0x97, 0x71, 0xd1, 0xa4, 0x31, 0x8a, 0x7a, 0x0b, 0x1c, 0xf5, 0xd6, 0xe1, 0x74, 0x79, 0x3a, 0x80, - 0x13, 0xf1, 0x7a, 0x5e, 0x64, 0x27, 0xce, 0xa8, 0xdf, 0x9e, 0x38, 0x95, 0x1f, 0x41, 0x45, 0x0d, - 0x27, 0xe6, 0x3e, 0x74, 0x3c, 0x85, 0xc2, 0x11, 0xd2, 0x5e, 0xbf, 0xfd, 0x04, 0xe2, 0x87, 0xdd, - 0x7a, 0x3c, 0x72, 0xf7, 0x41, 0x1e, 0x9b, 0x76, 0xec, 0x07, 0x20, 0xf1, 0xb2, 0x7b, 0x18, 0x27, - 0xed, 0x90, 0x24, 0xf9, 0x70, 0x7e, 0x91, 0x8f, 0x97, 0x19, 0x7e, 0x87, 0xee, 0x01, 0x53, 0x56, - 0x0c, 0x59, 0x7e, 0x7b, 0xf4, 0x84, 0x92, 0x9f, 0xd8, 0xfd, 0x61, 0xeb, 0x31, 0x49, 0x05, 0xe5, - 0x90, 0xc6, 0xe7, 0x07, 0xb0, 0x8b, 0x93, 0x90, 0x1a, 0x49, 0x24, 0x67, 0xbc, 0x17, 0xc4, 0x41, - 0x1e, 0x4c, 0x42, 0xec, 0x23, 0xd9, 0x14, 0xcc, 0x05, 0xe5, 0x44, 0x20, 0x93, 0x10, 0xc2, 0xa0, - 0xdf, 0x9b, 0xf7, 0x9d, 0x92, 0x48, 0x6e, 0x36, 0x9b, 0x98, 0xe5, 0xe1, 0x4f, 0x55, 0xd7, 0x1a, - 0x9f, 0x13, 0xb9, 0xe3, 0x91, 0x6d, 0x63, 0x78, 0x98, 0x5d, 0x5d, 0x7c, 0x40, 0x59, 0xcd, 0x03, - 0x35, 0xe0, 0x68, 0xfc, 0x58, 0xd6, 0x17, 0x46, 0x44, 0xd5, 0x59, 0xfe, 0x5d, 0x79, 0x17, 0xb1, - 0xe8, 0x77, 0x87, 0x2b, 0x12, 0xcb, 0xb5, 0xd1, 0xd8, 0x83, 0x82, 0xaf, 0x75, 0x96, 0xc1, 0x8f, - 0x64, 0x76, 0x89, 0x33, 0x98, 0x1b, 0x71, 0x53, 0x63, 0x43, 0x2c, 0x0c, 0x1c, 0xf7, 0xfe, 0xbf, - 0x75, 0x24, 0xf7, 0xd8, 0x95, 0xe9, 0x43, 0x9b, 0x6b, 0xdc, 0x96, 0xdf, 0xdc, 0x06, 0x0e, 0x77, - 0xc8, 0x5d, 0xd9, 0x78, 0x20, 0x74, 0xc5, 0x0c, 0x15, 0x99, 0x71, 0x56, 0x3a, 0xf2, 0x23, 0x9f, - 0x67, 0x85, 0xf6, 0x7d, 0xc8, 0x88, 0xdc, 0x99, 0x31, 0x46, 0x8e, 0xe5, 0x17, 0x53, 0xcf, 0x08, - 0x85, 0xba, 0x69, 0x23, 0x18, 0x6b, 0xa2, 0x8d, 0x9c, 0xc8, 0x8f, 0xa9, 0xaa, 0x91, 0xd5, 0x7a, - 0x68, 0x55, 0x4e, 0xcf, 0x74, 0xa5, 0x2b, 0xbf, 0x85, 0x7a, 0x56, 0x64, 0x00, 0x9c, 0x61, 0x26, - 0xde, 0x99, 0x0f, 0xe4, 0x67, 0xd2, 0x80, 0x99, 0x29, 0x6f, 0xc6, 0x67, 0x7a, 0xf3, 0x44, 0x7e, - 0x93, 0xf4, 0xcc, 0xec, 0xcf, 0xf2, 0x66, 0x7c, 0xa6, 0x37, 0x3d, 0xf9, 0xb5, 0x32, 0x60, 0x06, - 0xbc, 0xd9, 0x12, 0x86, 0xfa, 0xaa, 0x39, 0x4f, 0xc4, 0xd8, 0xe9, 0xcb, 0x6f, 0xd0, 0xfe, 0xcb, - 0x96, 0x94, 0x59, 0x86, 0xe2, 0x1d, 0x1a, 0xc8, 0xcf, 0xd3, 0x41, 0x43, 0xe0, 0xd1, 0xb6, 0xb8, - 0xa0, 0x4e, 0xec, 0x1c, 0x2e, 0x39, 0x60, 0xa9, 0x64, 0x2d, 0xf8, 0x53, 0x63, 0xce, 0x4c, 0x53, - 0xf1, 0x4e, 0x0d, 0xc1, 0x94, 0x3e, 0x65, 0x0a, 0xbc, 0xba, 0x2f, 0x4a, 0x8a, 0xa9, 0x43, 0xca, - 0xd0, 0xd1, 0x66, 0x9e, 0xca, 0xff, 0x4f, 0xc0, 0x33, 0x83, 0x19, 0x3d, 0xfc, 0xc6, 0x38, 0xc7, - 0x45, 0x1b, 0x19, 0xc9, 0x8f, 0xdc, 0xbe, 0x2f, 0xc4, 0x08, 0x6d, 0x09, 0xaa, 0xb4, 0xe3, 0xac, - 0x8c, 0xe5, 0xe7, 0x6f, 0xdf, 0x15, 0x24, 0x34, 0xba, 0x81, 0xe9, 0xd8, 0x98, 0xe4, 0x62, 0x6c, - 0x4c, 0x28, 0x22, 0xbf, 0x1e, 0x09, 0x58, 0x52, 0xaf, 0x42, 0x94, 0x69, 0xe3, 0x23, 0xbc, 0x84, - 0xf9, 0xf3, 0x07, 0xa4, 0x8f, 0x34, 0x59, 0x17, 0xd7, 0x96, 0xb0, 0x74, 0xb6, 0xe6, 0x8e, 0x02, - 0x71, 0x69, 0x43, 0xcc, 0x9d, 0x3b, 0x28, 0x7d, 0xac, 0xc9, 0xea, 0x12, 0x2d, 0x59, 0xc5, 0xa3, - 0x60, 0x64, 0x9a, 0x3b, 0x77, 0x58, 0xfa, 0x44, 0x93, 0x57, 0x11, 0x75, 0xd3, 0x33, 0xe2, 0x46, - 0xa6, 0xb9, 0x73, 0x87, 0xa5, 0xaf, 0xca, 0xda, 0x31, 0x51, 0xaf, 0xa9, 0x46, 0x28, 0x16, 0xcc, - 0x9f, 0x3f, 0x2c, 0x7d, 0x4d, 0xa3, 0x6b, 0x89, 0x44, 0xbd, 0xee, 0xad, 0x8b, 0x17, 0x99, 0xe6, - 0xcf, 0x1f, 0x96, 0xbe, 0xae, 0xd1, 0xe5, 0x45, 0xa2, 0xbe, 0x12, 0x30, 0x13, 0xf4, 0xe6, 0xec, - 0xb0, 0xf4, 0x0d, 0x8d, 0xee, 0x13, 0x12, 0xf5, 0x55, 0xcf, 0xcc, 0xfe, 0x94, 0x37, 0x67, 0x87, - 0xa5, 0x6f, 0xd2, 0x29, 0x1e, 0xcc, 0xac, 0x05, 0xcc, 0x50, 0x64, 0x2a, 0xbd, 0x40, 0x58, 0xfa, - 0x96, 0x46, 0xd7, 0x3e, 0x89, 0xfa, 0x5d, 0xcb, 0x1d, 0xdd, 0x8f, 0x4c, 0xa5, 0x17, 0x08, 0x4b, - 0x9f, 0x6a, 0x74, 0x3b, 0x94, 0xa8, 0xdf, 0x0b, 0x1a, 0xa2, 0xc8, 0xa4, 0xbf, 0x48, 0x58, 0xfa, - 0x36, 0x5a, 0x2a, 0x35, 0x12, 0x2b, 0xcb, 0x96, 0xeb, 0x80, 0x12, 0x99, 0xf4, 0x17, 0x09, 0x4b, - 0xdf, 0x41, 0x53, 0x3a, 0x98, 0xaa, 0x86, 0x4c, 0x81, 0x57, 0x0f, 0x44, 0xf1, 0xbc, 0x61, 0xe9, - 0xbb, 0xea, 0xad, 0x5b, 0xe1, 0x48, 0x89, 0x4d, 0x7b, 0xca, 0x3b, 0x3b, 0x33, 0x30, 0x7d, 0x8f, - 0x6a, 0x9c, 0xc6, 0xdc, 0x17, 0xe5, 0xcd, 0x94, 0x24, 0xf8, 0xaf, 0x4f, 0x86, 0xa9, 0x1d, 0x7f, - 0x7f, 0x9c, 0x19, 0xa3, 0xbe, 0xaf, 0xd1, 0xf5, 0x55, 0x91, 0x0d, 0x12, 0xde, 0xdb, 0x29, 0x32, - 0x60, 0x7d, 0xe0, 0xcf, 0xf2, 0xac, 0x68, 0xf5, 0x03, 0xed, 0x45, 0xc2, 0x55, 0x03, 0x6f, 0x6b, - 0xbd, 0xc5, 0xa0, 0x96, 0xcf, 0x8b, 0xd4, 0x33, 0x73, 0xb9, 0xaa, 0x1e, 0xc9, 0xd4, 0x5b, 0x5b, - 0x19, 0xa4, 0x0a, 0x66, 0x49, 0xb9, 0xd8, 0xc6, 0x6b, 0x5b, 0x8b, 0x58, 0xcc, 0x36, 0x23, 0xd9, - 0x1f, 0xc7, 0xb0, 0x4d, 0x66, 0xd7, 0x22, 0xd9, 0x9f, 0xc4, 0xb0, 0x6b, 0xcc, 0xae, 0x47, 0xb2, - 0xbf, 0x1a, 0xc3, 0xae, 0x33, 0x7b, 0x25, 0x92, 0xfd, 0xb5, 0x18, 0xf6, 0x0a, 0xb3, 0x57, 0x23, - 0xd9, 0x5f, 0x8f, 0x61, 0xaf, 0x32, 0x7b, 0x2d, 0x92, 0xfd, 0x8d, 0x18, 0xf6, 0x1a, 0xb3, 0xef, - 0x46, 0xb2, 0xbf, 0x19, 0xc3, 0xbe, 0xcb, 0xec, 0x7b, 0x91, 0xec, 0x6f, 0xc5, 0xb0, 0xef, 0x49, - 0x76, 0x75, 0x39, 0x92, 0xfd, 0x69, 0x34, 0xbb, 0xba, 0xcc, 0xec, 0x68, 0xad, 0x7d, 0x3b, 0x86, - 0xcd, 0x5a, 0xab, 0x46, 0x6b, 0xed, 0x3b, 0x31, 0x6c, 0xd6, 0x5a, 0x35, 0x5a, 0x6b, 0xdf, 0x8d, - 0x61, 0xb3, 0xd6, 0xaa, 0xd1, 0x5a, 0xfb, 0x5e, 0x0c, 0x9b, 0xb5, 0x56, 0x8d, 0xd6, 0xda, 0xf7, - 0x63, 0xd8, 0xac, 0xb5, 0x6a, 0xb4, 0xd6, 0x7e, 0x10, 0xc3, 0x66, 0xad, 0x55, 0xa3, 0xb5, 0xf6, - 0x7b, 0x31, 0x6c, 0xd6, 0x5a, 0x35, 0x5a, 0x6b, 0xbf, 0x1f, 0xc3, 0x66, 0xad, 0x55, 0xa3, 0xb5, - 0xf6, 0x07, 0x31, 0x6c, 0xd6, 0x9a, 0x19, 0xad, 0xb5, 0x3f, 0x8c, 0x66, 0x9b, 0xac, 0x35, 0x33, - 0x5a, 0x6b, 0x7f, 0x14, 0xc3, 0x66, 0xad, 0x99, 0xd1, 0x5a, 0xfb, 0xe3, 0x18, 0x36, 0x6b, 0xcd, - 0x8c, 0xd6, 0xda, 0x0f, 0x63, 0xd8, 0xac, 0x35, 0x33, 0x5a, 0x6b, 0x7f, 0x12, 0xc3, 0x66, 0xad, - 0x99, 0xd1, 0x5a, 0xfb, 0xd3, 0x18, 0x36, 0x6b, 0xcd, 0x8c, 0xd6, 0xda, 0x9f, 0xc5, 0xb0, 0x59, - 0x6b, 0x66, 0xb4, 0xd6, 0xfe, 0x3c, 0x86, 0xcd, 0x5a, 0x33, 0xa3, 0xb5, 0xf6, 0x17, 0x31, 0x6c, - 0xd6, 0x9a, 0x19, 0xad, 0xb5, 0xbf, 0x8c, 0x61, 0xb3, 0xd6, 0x6a, 0xd1, 0x5a, 0xfb, 0xab, 0x68, - 0x76, 0x8d, 0xb5, 0x56, 0x8b, 0xd6, 0xda, 0x5f, 0xc7, 0xb0, 0x59, 0x6b, 0xb5, 0x68, 0xad, 0xfd, - 0x4d, 0x0c, 0x9b, 0xb5, 0x56, 0x8b, 0xd6, 0xda, 0xdf, 0xc6, 0xb0, 0x59, 0x6b, 0xb5, 0x68, 0xad, - 0xfd, 0x28, 0x86, 0xcd, 0x5a, 0xab, 0x45, 0x6b, 0xed, 0xef, 0x62, 0xd8, 0xac, 0xb5, 0x5a, 0xb4, - 0xd6, 0xfe, 0x3e, 0x86, 0xcd, 0x5a, 0xab, 0x45, 0x6b, 0xed, 0x1f, 0x62, 0xd8, 0xac, 0xb5, 0x5a, - 0xb4, 0xd6, 0xfe, 0x31, 0x86, 0xcd, 0x5a, 0xab, 0x45, 0x6b, 0xed, 0x9f, 0x62, 0xd8, 0xac, 0xb5, - 0x7a, 0xb4, 0xd6, 0xfe, 0x39, 0x9a, 0x5d, 0x67, 0xad, 0xd5, 0xa3, 0xb5, 0xf6, 0x2f, 0x31, 0x6c, - 0xd6, 0x5a, 0x3d, 0x5a, 0x6b, 0xff, 0x1a, 0xc3, 0x66, 0xad, 0xd5, 0xa3, 0xb5, 0xf6, 0x6f, 0x31, - 0x6c, 0xd6, 0x5a, 0x3d, 0x5a, 0x6b, 0xff, 0x1e, 0xc3, 0x66, 0xad, 0xd5, 0xa3, 0xb5, 0xf6, 0x1f, - 0x31, 0x6c, 0xd6, 0x5a, 0x3d, 0x5a, 0x6b, 0x3f, 0x8e, 0x61, 0xb3, 0xd6, 0xea, 0xd1, 0x5a, 0xfb, - 0x49, 0x0c, 0x9b, 0xb5, 0x56, 0x8f, 0xd6, 0xda, 0x7f, 0xc6, 0xb0, 0x59, 0x6b, 0xf5, 0x68, 0xad, - 0xfd, 0x57, 0x0c, 0x9b, 0xb5, 0xb6, 0x12, 0xad, 0xb5, 0xff, 0x8e, 0x66, 0xaf, 0x2c, 0xff, 0x5f, - 0x00, 0x00, 0x00, 0xff, 0xff, 0x53, 0x6a, 0x15, 0xef, 0x13, 0x38, 0x00, 0x00, -} diff --git a/vendor/github.com/golang/protobuf/proto/testdata/test.proto b/vendor/github.com/golang/protobuf/proto/testdata/test.proto deleted file mode 100644 index 698c8ce2..00000000 --- a/vendor/github.com/golang/protobuf/proto/testdata/test.proto +++ /dev/null @@ -1,535 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2010 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// A feature-rich test file for the protocol compiler and libraries. - -syntax = "proto2"; - -package testdata; - -enum FOO { FOO1 = 1; }; - -message GoEnum { - required FOO foo = 1; -} - -message GoTestField { - required string Label = 1; - required string Type = 2; -} - -message GoTest { - // An enum, for completeness. - enum KIND { - VOID = 0; - - // Basic types - BOOL = 1; - BYTES = 2; - FINGERPRINT = 3; - FLOAT = 4; - INT = 5; - STRING = 6; - TIME = 7; - - // Groupings - TUPLE = 8; - ARRAY = 9; - MAP = 10; - - // Table types - TABLE = 11; - - // Functions - FUNCTION = 12; // last tag - }; - - // Some typical parameters - required KIND Kind = 1; - optional string Table = 2; - optional int32 Param = 3; - - // Required, repeated and optional foreign fields. - required GoTestField RequiredField = 4; - repeated GoTestField RepeatedField = 5; - optional GoTestField OptionalField = 6; - - // Required fields of all basic types - required bool F_Bool_required = 10; - required int32 F_Int32_required = 11; - required int64 F_Int64_required = 12; - required fixed32 F_Fixed32_required = 13; - required fixed64 F_Fixed64_required = 14; - required uint32 F_Uint32_required = 15; - required uint64 F_Uint64_required = 16; - required float F_Float_required = 17; - required double F_Double_required = 18; - required string F_String_required = 19; - required bytes F_Bytes_required = 101; - required sint32 F_Sint32_required = 102; - required sint64 F_Sint64_required = 103; - - // Repeated fields of all basic types - repeated bool F_Bool_repeated = 20; - repeated int32 F_Int32_repeated = 21; - repeated int64 F_Int64_repeated = 22; - repeated fixed32 F_Fixed32_repeated = 23; - repeated fixed64 F_Fixed64_repeated = 24; - repeated uint32 F_Uint32_repeated = 25; - repeated uint64 F_Uint64_repeated = 26; - repeated float F_Float_repeated = 27; - repeated double F_Double_repeated = 28; - repeated string F_String_repeated = 29; - repeated bytes F_Bytes_repeated = 201; - repeated sint32 F_Sint32_repeated = 202; - repeated sint64 F_Sint64_repeated = 203; - - // Optional fields of all basic types - optional bool F_Bool_optional = 30; - optional int32 F_Int32_optional = 31; - optional int64 F_Int64_optional = 32; - optional fixed32 F_Fixed32_optional = 33; - optional fixed64 F_Fixed64_optional = 34; - optional uint32 F_Uint32_optional = 35; - optional uint64 F_Uint64_optional = 36; - optional float F_Float_optional = 37; - optional double F_Double_optional = 38; - optional string F_String_optional = 39; - optional bytes F_Bytes_optional = 301; - optional sint32 F_Sint32_optional = 302; - optional sint64 F_Sint64_optional = 303; - - // Default-valued fields of all basic types - optional bool F_Bool_defaulted = 40 [default=true]; - optional int32 F_Int32_defaulted = 41 [default=32]; - optional int64 F_Int64_defaulted = 42 [default=64]; - optional fixed32 F_Fixed32_defaulted = 43 [default=320]; - optional fixed64 F_Fixed64_defaulted = 44 [default=640]; - optional uint32 F_Uint32_defaulted = 45 [default=3200]; - optional uint64 F_Uint64_defaulted = 46 [default=6400]; - optional float F_Float_defaulted = 47 [default=314159.]; - optional double F_Double_defaulted = 48 [default=271828.]; - optional string F_String_defaulted = 49 [default="hello, \"world!\"\n"]; - optional bytes F_Bytes_defaulted = 401 [default="Bignose"]; - optional sint32 F_Sint32_defaulted = 402 [default = -32]; - optional sint64 F_Sint64_defaulted = 403 [default = -64]; - - // Packed repeated fields (no string or bytes). - repeated bool F_Bool_repeated_packed = 50 [packed=true]; - repeated int32 F_Int32_repeated_packed = 51 [packed=true]; - repeated int64 F_Int64_repeated_packed = 52 [packed=true]; - repeated fixed32 F_Fixed32_repeated_packed = 53 [packed=true]; - repeated fixed64 F_Fixed64_repeated_packed = 54 [packed=true]; - repeated uint32 F_Uint32_repeated_packed = 55 [packed=true]; - repeated uint64 F_Uint64_repeated_packed = 56 [packed=true]; - repeated float F_Float_repeated_packed = 57 [packed=true]; - repeated double F_Double_repeated_packed = 58 [packed=true]; - repeated sint32 F_Sint32_repeated_packed = 502 [packed=true]; - repeated sint64 F_Sint64_repeated_packed = 503 [packed=true]; - - // Required, repeated, and optional groups. - required group RequiredGroup = 70 { - required string RequiredField = 71; - }; - - repeated group RepeatedGroup = 80 { - required string RequiredField = 81; - }; - - optional group OptionalGroup = 90 { - required string RequiredField = 91; - }; -} - -// For testing skipping of unrecognized fields. -// Numbers are all big, larger than tag numbers in GoTestField, -// the message used in the corresponding test. -message GoSkipTest { - required int32 skip_int32 = 11; - required fixed32 skip_fixed32 = 12; - required fixed64 skip_fixed64 = 13; - required string skip_string = 14; - required group SkipGroup = 15 { - required int32 group_int32 = 16; - required string group_string = 17; - } -} - -// For testing packed/non-packed decoder switching. -// A serialized instance of one should be deserializable as the other. -message NonPackedTest { - repeated int32 a = 1; -} - -message PackedTest { - repeated int32 b = 1 [packed=true]; -} - -message MaxTag { - // Maximum possible tag number. - optional string last_field = 536870911; -} - -message OldMessage { - message Nested { - optional string name = 1; - } - optional Nested nested = 1; - - optional int32 num = 2; -} - -// NewMessage is wire compatible with OldMessage; -// imagine it as a future version. -message NewMessage { - message Nested { - optional string name = 1; - optional string food_group = 2; - } - optional Nested nested = 1; - - // This is an int32 in OldMessage. - optional int64 num = 2; -} - -// Smaller tests for ASCII formatting. - -message InnerMessage { - required string host = 1; - optional int32 port = 2 [default=4000]; - optional bool connected = 3; -} - -message OtherMessage { - optional int64 key = 1; - optional bytes value = 2; - optional float weight = 3; - optional InnerMessage inner = 4; - - extensions 100 to max; -} - -message MyMessage { - required int32 count = 1; - optional string name = 2; - optional string quote = 3; - repeated string pet = 4; - optional InnerMessage inner = 5; - repeated OtherMessage others = 6; - repeated InnerMessage rep_inner = 12; - - enum Color { - RED = 0; - GREEN = 1; - BLUE = 2; - }; - optional Color bikeshed = 7; - - optional group SomeGroup = 8 { - optional int32 group_field = 9; - } - - // This field becomes [][]byte in the generated code. - repeated bytes rep_bytes = 10; - - optional double bigfloat = 11; - - extensions 100 to max; -} - -message Ext { - extend MyMessage { - optional Ext more = 103; - optional string text = 104; - optional int32 number = 105; - } - - optional string data = 1; -} - -extend MyMessage { - repeated string greeting = 106; -} - -message ComplexExtension { - optional int32 first = 1; - optional int32 second = 2; - repeated int32 third = 3; -} - -extend OtherMessage { - optional ComplexExtension complex = 200; - repeated ComplexExtension r_complex = 201; -} - -message DefaultsMessage { - enum DefaultsEnum { - ZERO = 0; - ONE = 1; - TWO = 2; - }; - extensions 100 to max; -} - -extend DefaultsMessage { - optional double no_default_double = 101; - optional float no_default_float = 102; - optional int32 no_default_int32 = 103; - optional int64 no_default_int64 = 104; - optional uint32 no_default_uint32 = 105; - optional uint64 no_default_uint64 = 106; - optional sint32 no_default_sint32 = 107; - optional sint64 no_default_sint64 = 108; - optional fixed32 no_default_fixed32 = 109; - optional fixed64 no_default_fixed64 = 110; - optional sfixed32 no_default_sfixed32 = 111; - optional sfixed64 no_default_sfixed64 = 112; - optional bool no_default_bool = 113; - optional string no_default_string = 114; - optional bytes no_default_bytes = 115; - optional DefaultsMessage.DefaultsEnum no_default_enum = 116; - - optional double default_double = 201 [default = 3.1415]; - optional float default_float = 202 [default = 3.14]; - optional int32 default_int32 = 203 [default = 42]; - optional int64 default_int64 = 204 [default = 43]; - optional uint32 default_uint32 = 205 [default = 44]; - optional uint64 default_uint64 = 206 [default = 45]; - optional sint32 default_sint32 = 207 [default = 46]; - optional sint64 default_sint64 = 208 [default = 47]; - optional fixed32 default_fixed32 = 209 [default = 48]; - optional fixed64 default_fixed64 = 210 [default = 49]; - optional sfixed32 default_sfixed32 = 211 [default = 50]; - optional sfixed64 default_sfixed64 = 212 [default = 51]; - optional bool default_bool = 213 [default = true]; - optional string default_string = 214 [default = "Hello, string"]; - optional bytes default_bytes = 215 [default = "Hello, bytes"]; - optional DefaultsMessage.DefaultsEnum default_enum = 216 [default = ONE]; -} - -message MyMessageSet { - option message_set_wire_format = true; - extensions 100 to max; -} - -message Empty { -} - -extend MyMessageSet { - optional Empty x201 = 201; - optional Empty x202 = 202; - optional Empty x203 = 203; - optional Empty x204 = 204; - optional Empty x205 = 205; - optional Empty x206 = 206; - optional Empty x207 = 207; - optional Empty x208 = 208; - optional Empty x209 = 209; - optional Empty x210 = 210; - optional Empty x211 = 211; - optional Empty x212 = 212; - optional Empty x213 = 213; - optional Empty x214 = 214; - optional Empty x215 = 215; - optional Empty x216 = 216; - optional Empty x217 = 217; - optional Empty x218 = 218; - optional Empty x219 = 219; - optional Empty x220 = 220; - optional Empty x221 = 221; - optional Empty x222 = 222; - optional Empty x223 = 223; - optional Empty x224 = 224; - optional Empty x225 = 225; - optional Empty x226 = 226; - optional Empty x227 = 227; - optional Empty x228 = 228; - optional Empty x229 = 229; - optional Empty x230 = 230; - optional Empty x231 = 231; - optional Empty x232 = 232; - optional Empty x233 = 233; - optional Empty x234 = 234; - optional Empty x235 = 235; - optional Empty x236 = 236; - optional Empty x237 = 237; - optional Empty x238 = 238; - optional Empty x239 = 239; - optional Empty x240 = 240; - optional Empty x241 = 241; - optional Empty x242 = 242; - optional Empty x243 = 243; - optional Empty x244 = 244; - optional Empty x245 = 245; - optional Empty x246 = 246; - optional Empty x247 = 247; - optional Empty x248 = 248; - optional Empty x249 = 249; - optional Empty x250 = 250; -} - -message MessageList { - repeated group Message = 1 { - required string name = 2; - required int32 count = 3; - } -} - -message Strings { - optional string string_field = 1; - optional bytes bytes_field = 2; -} - -message Defaults { - enum Color { - RED = 0; - GREEN = 1; - BLUE = 2; - } - - // Default-valued fields of all basic types. - // Same as GoTest, but copied here to make testing easier. - optional bool F_Bool = 1 [default=true]; - optional int32 F_Int32 = 2 [default=32]; - optional int64 F_Int64 = 3 [default=64]; - optional fixed32 F_Fixed32 = 4 [default=320]; - optional fixed64 F_Fixed64 = 5 [default=640]; - optional uint32 F_Uint32 = 6 [default=3200]; - optional uint64 F_Uint64 = 7 [default=6400]; - optional float F_Float = 8 [default=314159.]; - optional double F_Double = 9 [default=271828.]; - optional string F_String = 10 [default="hello, \"world!\"\n"]; - optional bytes F_Bytes = 11 [default="Bignose"]; - optional sint32 F_Sint32 = 12 [default=-32]; - optional sint64 F_Sint64 = 13 [default=-64]; - optional Color F_Enum = 14 [default=GREEN]; - - // More fields with crazy defaults. - optional float F_Pinf = 15 [default=inf]; - optional float F_Ninf = 16 [default=-inf]; - optional float F_Nan = 17 [default=nan]; - - // Sub-message. - optional SubDefaults sub = 18; - - // Redundant but explicit defaults. - optional string str_zero = 19 [default=""]; -} - -message SubDefaults { - optional int64 n = 1 [default=7]; -} - -message RepeatedEnum { - enum Color { - RED = 1; - } - repeated Color color = 1; -} - -message MoreRepeated { - repeated bool bools = 1; - repeated bool bools_packed = 2 [packed=true]; - repeated int32 ints = 3; - repeated int32 ints_packed = 4 [packed=true]; - repeated int64 int64s_packed = 7 [packed=true]; - repeated string strings = 5; - repeated fixed32 fixeds = 6; -} - -// GroupOld and GroupNew have the same wire format. -// GroupNew has a new field inside a group. - -message GroupOld { - optional group G = 101 { - optional int32 x = 2; - } -} - -message GroupNew { - optional group G = 101 { - optional int32 x = 2; - optional int32 y = 3; - } -} - -message FloatingPoint { - required double f = 1; -} - -message MessageWithMap { - map name_mapping = 1; - map msg_mapping = 2; - map byte_mapping = 3; - map str_to_str = 4; -} - -message Oneof { - oneof union { - bool F_Bool = 1; - int32 F_Int32 = 2; - int64 F_Int64 = 3; - fixed32 F_Fixed32 = 4; - fixed64 F_Fixed64 = 5; - uint32 F_Uint32 = 6; - uint64 F_Uint64 = 7; - float F_Float = 8; - double F_Double = 9; - string F_String = 10; - bytes F_Bytes = 11; - sint32 F_Sint32 = 12; - sint64 F_Sint64 = 13; - MyMessage.Color F_Enum = 14; - GoTestField F_Message = 15; - group F_Group = 16 { - optional int32 x = 17; - } - int32 F_Largest_Tag = 536870911; - } - - oneof tormato { - int32 value = 100; - } -} - -message Communique { - optional bool make_me_cry = 1; - - // This is a oneof, called "union". - oneof union { - int32 number = 5; - string name = 6; - bytes data = 7; - double temp_c = 8; - MyMessage.Color col = 9; - Strings msg = 10; - } -} diff --git a/vendor/github.com/golang/protobuf/proto/text.go b/vendor/github.com/golang/protobuf/proto/text.go index 1cbaf86d..965876bf 100644 --- a/vendor/github.com/golang/protobuf/proto/text.go +++ b/vendor/github.com/golang/protobuf/proto/text.go @@ -154,7 +154,7 @@ func (w *textWriter) indent() { w.ind++ } func (w *textWriter) unindent() { if w.ind == 0 { - log.Printf("proto: textWriter unindented too far") + log.Print("proto: textWriter unindented too far") return } w.ind-- @@ -175,7 +175,93 @@ type raw interface { Bytes() []byte } -func writeStruct(w *textWriter, sv reflect.Value) error { +func requiresQuotes(u string) bool { + // When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted. + for _, ch := range u { + switch { + case ch == '.' || ch == '/' || ch == '_': + continue + case '0' <= ch && ch <= '9': + continue + case 'A' <= ch && ch <= 'Z': + continue + case 'a' <= ch && ch <= 'z': + continue + default: + return true + } + } + return false +} + +// isAny reports whether sv is a google.protobuf.Any message +func isAny(sv reflect.Value) bool { + type wkt interface { + XXX_WellKnownType() string + } + t, ok := sv.Addr().Interface().(wkt) + return ok && t.XXX_WellKnownType() == "Any" +} + +// writeProto3Any writes an expanded google.protobuf.Any message. +// +// It returns (false, nil) if sv value can't be unmarshaled (e.g. because +// required messages are not linked in). +// +// It returns (true, error) when sv was written in expanded format or an error +// was encountered. +func (tm *TextMarshaler) writeProto3Any(w *textWriter, sv reflect.Value) (bool, error) { + turl := sv.FieldByName("TypeUrl") + val := sv.FieldByName("Value") + if !turl.IsValid() || !val.IsValid() { + return true, errors.New("proto: invalid google.protobuf.Any message") + } + + b, ok := val.Interface().([]byte) + if !ok { + return true, errors.New("proto: invalid google.protobuf.Any message") + } + + parts := strings.Split(turl.String(), "/") + mt := MessageType(parts[len(parts)-1]) + if mt == nil { + return false, nil + } + m := reflect.New(mt.Elem()) + if err := Unmarshal(b, m.Interface().(Message)); err != nil { + return false, nil + } + w.Write([]byte("[")) + u := turl.String() + if requiresQuotes(u) { + writeString(w, u) + } else { + w.Write([]byte(u)) + } + if w.compact { + w.Write([]byte("]:<")) + } else { + w.Write([]byte("]: <\n")) + w.ind++ + } + if err := tm.writeStruct(w, m.Elem()); err != nil { + return true, err + } + if w.compact { + w.Write([]byte("> ")) + } else { + w.ind-- + w.Write([]byte(">\n")) + } + return true, nil +} + +func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { + if tm.ExpandAny && isAny(sv) { + if canExpand, err := tm.writeProto3Any(w, sv); canExpand { + return err + } + } st := sv.Type() sprops := GetProperties(st) for i := 0; i < sv.NumField(); i++ { @@ -227,7 +313,7 @@ func writeStruct(w *textWriter, sv reflect.Value) error { } continue } - if err := writeAny(w, v, props); err != nil { + if err := tm.writeAny(w, v, props); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -269,7 +355,7 @@ func writeStruct(w *textWriter, sv reflect.Value) error { return err } } - if err := writeAny(w, key, props.mkeyprop); err != nil { + if err := tm.writeAny(w, key, props.mkeyprop); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -286,7 +372,7 @@ func writeStruct(w *textWriter, sv reflect.Value) error { return err } } - if err := writeAny(w, val, props.mvalprop); err != nil { + if err := tm.writeAny(w, val, props.mvalprop); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -358,7 +444,7 @@ func writeStruct(w *textWriter, sv reflect.Value) error { } // Enums have a String method, so writeAny will work fine. - if err := writeAny(w, fv, props); err != nil { + if err := tm.writeAny(w, fv, props); err != nil { return err } @@ -369,8 +455,8 @@ func writeStruct(w *textWriter, sv reflect.Value) error { // Extensions (the XXX_extensions field). pv := sv.Addr() - if pv.Type().Implements(extendableProtoType) { - if err := writeExtensions(w, pv); err != nil { + if _, ok := extendable(pv.Interface()); ok { + if err := tm.writeExtensions(w, pv); err != nil { return err } } @@ -400,7 +486,7 @@ func writeRaw(w *textWriter, b []byte) error { } // writeAny writes an arbitrary field. -func writeAny(w *textWriter, v reflect.Value, props *Properties) error { +func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { v = reflect.Indirect(v) // Floats have special cases. @@ -427,7 +513,7 @@ func writeAny(w *textWriter, v reflect.Value, props *Properties) error { switch v.Kind() { case reflect.Slice: // Should only be a []byte; repeated fields are handled in writeStruct. - if err := writeString(w, string(v.Interface().([]byte))); err != nil { + if err := writeString(w, string(v.Bytes())); err != nil { return err } case reflect.String: @@ -449,15 +535,15 @@ func writeAny(w *textWriter, v reflect.Value, props *Properties) error { } } w.indent() - if tm, ok := v.Interface().(encoding.TextMarshaler); ok { - text, err := tm.MarshalText() + if etm, ok := v.Interface().(encoding.TextMarshaler); ok { + text, err := etm.MarshalText() if err != nil { return err } if _, err = w.Write(text); err != nil { return err } - } else if err := writeStruct(w, v); err != nil { + } else if err := tm.writeStruct(w, v); err != nil { return err } w.unindent() @@ -601,19 +687,24 @@ func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // writeExtensions writes all the extensions in pv. // pv is assumed to be a pointer to a protocol message struct that is extendable. -func writeExtensions(w *textWriter, pv reflect.Value) error { +func (tm *TextMarshaler) writeExtensions(w *textWriter, pv reflect.Value) error { emap := extensionMaps[pv.Type().Elem()] - ep := pv.Interface().(extendableProto) + ep, _ := extendable(pv.Interface()) // Order the extensions by ID. // This isn't strictly necessary, but it will give us // canonical output, which will also make testing easier. - m := ep.ExtensionMap() + m, mu := ep.extensionsRead() + if m == nil { + return nil + } + mu.Lock() ids := make([]int32, 0, len(m)) for id := range m { ids = append(ids, id) } sort.Sort(int32Slice(ids)) + mu.Unlock() for _, extNum := range ids { ext := m[extNum] @@ -636,13 +727,13 @@ func writeExtensions(w *textWriter, pv reflect.Value) error { // Repeated extensions will appear as a slice. if !desc.repeated() { - if err := writeExtension(w, desc.Name, pb); err != nil { + if err := tm.writeExtension(w, desc.Name, pb); err != nil { return err } } else { v := reflect.ValueOf(pb) for i := 0; i < v.Len(); i++ { - if err := writeExtension(w, desc.Name, v.Index(i).Interface()); err != nil { + if err := tm.writeExtension(w, desc.Name, v.Index(i).Interface()); err != nil { return err } } @@ -651,7 +742,7 @@ func writeExtensions(w *textWriter, pv reflect.Value) error { return nil } -func writeExtension(w *textWriter, name string, pb interface{}) error { +func (tm *TextMarshaler) writeExtension(w *textWriter, name string, pb interface{}) error { if _, err := fmt.Fprintf(w, "[%s]:", name); err != nil { return err } @@ -660,7 +751,7 @@ func writeExtension(w *textWriter, name string, pb interface{}) error { return err } } - if err := writeAny(w, reflect.ValueOf(pb), nil); err != nil { + if err := tm.writeAny(w, reflect.ValueOf(pb), nil); err != nil { return err } if err := w.WriteByte('\n'); err != nil { @@ -687,12 +778,13 @@ func (w *textWriter) writeIndent() { // TextMarshaler is a configurable text format marshaler. type TextMarshaler struct { - Compact bool // use compact text format (one line). + Compact bool // use compact text format (one line). + ExpandAny bool // expand google.protobuf.Any messages of known types } // Marshal writes a given protocol buffer in text format. // The only errors returned are from w. -func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { +func (tm *TextMarshaler) Marshal(w io.Writer, pb Message) error { val := reflect.ValueOf(pb) if pb == nil || val.IsNil() { w.Write([]byte("")) @@ -707,11 +799,11 @@ func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { aw := &textWriter{ w: ww, complete: true, - compact: m.Compact, + compact: tm.Compact, } - if tm, ok := pb.(encoding.TextMarshaler); ok { - text, err := tm.MarshalText() + if etm, ok := pb.(encoding.TextMarshaler); ok { + text, err := etm.MarshalText() if err != nil { return err } @@ -725,7 +817,7 @@ func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { } // Dereference the received pointer so we don't have outer < and >. v := reflect.Indirect(val) - if err := writeStruct(aw, v); err != nil { + if err := tm.writeStruct(aw, v); err != nil { return err } if bw != nil { @@ -735,9 +827,9 @@ func (m *TextMarshaler) Marshal(w io.Writer, pb Message) error { } // Text is the same as Marshal, but returns the string directly. -func (m *TextMarshaler) Text(pb Message) string { +func (tm *TextMarshaler) Text(pb Message) string { var buf bytes.Buffer - m.Marshal(&buf, pb) + tm.Marshal(&buf, pb) return buf.String() } diff --git a/vendor/github.com/golang/protobuf/proto/text_parser.go b/vendor/github.com/golang/protobuf/proto/text_parser.go index 45132326..61f83c1e 100644 --- a/vendor/github.com/golang/protobuf/proto/text_parser.go +++ b/vendor/github.com/golang/protobuf/proto/text_parser.go @@ -44,6 +44,9 @@ import ( "unicode/utf8" ) +// Error string emitted when deserializing Any and fields are already set +const anyRepeatedlyUnpacked = "Any message unpacked multiple times, or %q already set" + type ParseError struct { Message string Line int // 1-based line number @@ -163,7 +166,7 @@ func (p *textParser) advance() { p.cur.offset, p.cur.line = p.offset, p.line p.cur.unquoted = "" switch p.s[0] { - case '<', '>', '{', '}', ':', '[', ']', ';', ',': + case '<', '>', '{', '}', ':', '[', ']', ';', ',', '/': // Single symbol p.cur.value, p.s = p.s[0:1], p.s[1:len(p.s)] case '"', '\'': @@ -451,7 +454,10 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { fieldSet := make(map[string]bool) // A struct is a sequence of "name: value", terminated by one of // '>' or '}', or the end of the input. A name may also be - // "[extension]". + // "[extension]" or "[type/url]". + // + // The whole struct can also be an expanded Any message, like: + // [type/url] < ... struct contents ... > for { tok := p.next() if tok.err != nil { @@ -461,33 +467,74 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { break } if tok.value == "[" { - // Looks like an extension. + // Looks like an extension or an Any. // // TODO: Check whether we need to handle // namespace rooted names (e.g. ".something.Foo"). - tok = p.next() - if tok.err != nil { - return tok.err + extName, err := p.consumeExtName() + if err != nil { + return err } + + if s := strings.LastIndex(extName, "/"); s >= 0 { + // If it contains a slash, it's an Any type URL. + messageName := extName[s+1:] + mt := MessageType(messageName) + if mt == nil { + return p.errorf("unrecognized message %q in google.protobuf.Any", messageName) + } + tok = p.next() + if tok.err != nil { + return tok.err + } + // consume an optional colon + if tok.value == ":" { + tok = p.next() + if tok.err != nil { + return tok.err + } + } + var terminator string + switch tok.value { + case "<": + terminator = ">" + case "{": + terminator = "}" + default: + return p.errorf("expected '{' or '<', found %q", tok.value) + } + v := reflect.New(mt.Elem()) + if pe := p.readStruct(v.Elem(), terminator); pe != nil { + return pe + } + b, err := Marshal(v.Interface().(Message)) + if err != nil { + return p.errorf("failed to marshal message of type %q: %v", messageName, err) + } + if fieldSet["type_url"] { + return p.errorf(anyRepeatedlyUnpacked, "type_url") + } + if fieldSet["value"] { + return p.errorf(anyRepeatedlyUnpacked, "value") + } + sv.FieldByName("TypeUrl").SetString(extName) + sv.FieldByName("Value").SetBytes(b) + fieldSet["type_url"] = true + fieldSet["value"] = true + continue + } + var desc *ExtensionDesc // This could be faster, but it's functional. // TODO: Do something smarter than a linear scan. for _, d := range RegisteredExtensions(reflect.New(st).Interface().(Message)) { - if d.Name == tok.value { + if d.Name == extName { desc = d break } } if desc == nil { - return p.errorf("unrecognized extension %q", tok.value) - } - // Check the extension terminator. - tok = p.next() - if tok.err != nil { - return tok.err - } - if tok.value != "]" { - return p.errorf("unrecognized extension terminator %q", tok.value) + return p.errorf("unrecognized extension %q", extName) } props := &Properties{} @@ -514,7 +561,7 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { } reqFieldErr = err } - ep := sv.Addr().Interface().(extendableProto) + ep := sv.Addr().Interface().(Message) if !rep { SetExtension(ep, desc, ext.Interface()) } else { @@ -545,7 +592,11 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { props = oop.Prop nv := reflect.New(oop.Type.Elem()) dst = nv.Elem().Field(0) - sv.Field(oop.Field).Set(nv) + field := sv.Field(oop.Field) + if !field.IsNil() { + return p.errorf("field '%s' would overwrite already parsed oneof '%s'", name, sv.Type().Field(oop.Field).Name) + } + field.Set(nv) } if !dst.IsValid() { return p.errorf("unknown field name %q in %v", name, st) @@ -566,8 +617,9 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { // The map entry should be this sequence of tokens: // < key : KEY value : VALUE > - // Technically the "key" and "value" could come in any order, - // but in practice they won't. + // However, implementations may omit key or value, and technically + // we should support them in any order. See b/28924776 for a time + // this went wrong. tok := p.next() var terminator string @@ -579,32 +631,39 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { default: return p.errorf("expected '{' or '<', found %q", tok.value) } - if err := p.consumeToken("key"); err != nil { - return err - } - if err := p.consumeToken(":"); err != nil { - return err - } - if err := p.readAny(key, props.mkeyprop); err != nil { - return err - } - if err := p.consumeOptionalSeparator(); err != nil { - return err - } - if err := p.consumeToken("value"); err != nil { - return err - } - if err := p.checkForColon(props.mvalprop, dst.Type().Elem()); err != nil { - return err - } - if err := p.readAny(val, props.mvalprop); err != nil { - return err - } - if err := p.consumeOptionalSeparator(); err != nil { - return err - } - if err := p.consumeToken(terminator); err != nil { - return err + for { + tok := p.next() + if tok.err != nil { + return tok.err + } + if tok.value == terminator { + break + } + switch tok.value { + case "key": + if err := p.consumeToken(":"); err != nil { + return err + } + if err := p.readAny(key, props.mkeyprop); err != nil { + return err + } + if err := p.consumeOptionalSeparator(); err != nil { + return err + } + case "value": + if err := p.checkForColon(props.mvalprop, dst.Type().Elem()); err != nil { + return err + } + if err := p.readAny(val, props.mvalprop); err != nil { + return err + } + if err := p.consumeOptionalSeparator(); err != nil { + return err + } + default: + p.back() + return p.errorf(`expected "key", "value", or %q, found %q`, terminator, tok.value) + } } dst.SetMapIndex(key, val) @@ -627,7 +686,8 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { return err } reqFieldErr = err - } else if props.Required { + } + if props.Required { reqCount-- } @@ -643,6 +703,35 @@ func (p *textParser) readStruct(sv reflect.Value, terminator string) error { return reqFieldErr } +// consumeExtName consumes extension name or expanded Any type URL and the +// following ']'. It returns the name or URL consumed. +func (p *textParser) consumeExtName() (string, error) { + tok := p.next() + if tok.err != nil { + return "", tok.err + } + + // If extension name or type url is quoted, it's a single token. + if len(tok.value) > 2 && isQuote(tok.value[0]) && tok.value[len(tok.value)-1] == tok.value[0] { + name, err := unquoteC(tok.value[1:len(tok.value)-1], rune(tok.value[0])) + if err != nil { + return "", err + } + return name, p.consumeToken("]") + } + + // Consume everything up to "]" + var parts []string + for tok.value != "]" { + parts = append(parts, tok.value) + tok = p.next() + if tok.err != nil { + return "", p.errorf("unrecognized type_url or extension name: %s", tok.err) + } + } + return strings.Join(parts, ""), nil +} + // consumeOptionalSeparator consumes an optional semicolon or comma. // It is used in readStruct to provide backward compatibility. func (p *textParser) consumeOptionalSeparator() error { @@ -707,12 +796,12 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error { fv.Set(reflect.Append(fv, reflect.New(at.Elem()).Elem())) return p.readAny(fv.Index(fv.Len()-1), props) case reflect.Bool: - // Either "true", "false", 1 or 0. + // true/1/t/True or false/f/0/False. switch tok.value { - case "true", "1": + case "true", "1", "t", "True": fv.SetBool(true) return nil - case "false", "0": + case "false", "0", "f", "False": fv.SetBool(false) return nil } diff --git a/vendor/github.com/golang/protobuf/proto/text_parser_test.go b/vendor/github.com/golang/protobuf/proto/text_parser_test.go deleted file mode 100644 index d2e4cca0..00000000 --- a/vendor/github.com/golang/protobuf/proto/text_parser_test.go +++ /dev/null @@ -1,557 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2010 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "math" - "reflect" - "testing" - - . "github.com/golang/protobuf/proto" - proto3pb "github.com/golang/protobuf/proto/proto3_proto" - . "github.com/golang/protobuf/proto/testdata" -) - -type UnmarshalTextTest struct { - in string - err string // if "", no error expected - out *MyMessage -} - -func buildExtStructTest(text string) UnmarshalTextTest { - msg := &MyMessage{ - Count: Int32(42), - } - SetExtension(msg, E_Ext_More, &Ext{ - Data: String("Hello, world!"), - }) - return UnmarshalTextTest{in: text, out: msg} -} - -func buildExtDataTest(text string) UnmarshalTextTest { - msg := &MyMessage{ - Count: Int32(42), - } - SetExtension(msg, E_Ext_Text, String("Hello, world!")) - SetExtension(msg, E_Ext_Number, Int32(1729)) - return UnmarshalTextTest{in: text, out: msg} -} - -func buildExtRepStringTest(text string) UnmarshalTextTest { - msg := &MyMessage{ - Count: Int32(42), - } - if err := SetExtension(msg, E_Greeting, []string{"bula", "hola"}); err != nil { - panic(err) - } - return UnmarshalTextTest{in: text, out: msg} -} - -var unMarshalTextTests = []UnmarshalTextTest{ - // Basic - { - in: " count:42\n name:\"Dave\" ", - out: &MyMessage{ - Count: Int32(42), - Name: String("Dave"), - }, - }, - - // Empty quoted string - { - in: `count:42 name:""`, - out: &MyMessage{ - Count: Int32(42), - Name: String(""), - }, - }, - - // Quoted string concatenation with double quotes - { - in: `count:42 name: "My name is "` + "\n" + `"elsewhere"`, - out: &MyMessage{ - Count: Int32(42), - Name: String("My name is elsewhere"), - }, - }, - - // Quoted string concatenation with single quotes - { - in: "count:42 name: 'My name is '\n'elsewhere'", - out: &MyMessage{ - Count: Int32(42), - Name: String("My name is elsewhere"), - }, - }, - - // Quoted string concatenations with mixed quotes - { - in: "count:42 name: 'My name is '\n\"elsewhere\"", - out: &MyMessage{ - Count: Int32(42), - Name: String("My name is elsewhere"), - }, - }, - { - in: "count:42 name: \"My name is \"\n'elsewhere'", - out: &MyMessage{ - Count: Int32(42), - Name: String("My name is elsewhere"), - }, - }, - - // Quoted string with escaped apostrophe - { - in: `count:42 name: "HOLIDAY - New Year\'s Day"`, - out: &MyMessage{ - Count: Int32(42), - Name: String("HOLIDAY - New Year's Day"), - }, - }, - - // Quoted string with single quote - { - in: `count:42 name: 'Roger "The Ramster" Ramjet'`, - out: &MyMessage{ - Count: Int32(42), - Name: String(`Roger "The Ramster" Ramjet`), - }, - }, - - // Quoted string with all the accepted special characters from the C++ test - { - in: `count:42 name: ` + "\"\\\"A string with \\' characters \\n and \\r newlines and \\t tabs and \\001 slashes \\\\ and multiple spaces\"", - out: &MyMessage{ - Count: Int32(42), - Name: String("\"A string with ' characters \n and \r newlines and \t tabs and \001 slashes \\ and multiple spaces"), - }, - }, - - // Quoted string with quoted backslash - { - in: `count:42 name: "\\'xyz"`, - out: &MyMessage{ - Count: Int32(42), - Name: String(`\'xyz`), - }, - }, - - // Quoted string with UTF-8 bytes. - { - in: "count:42 name: '\303\277\302\201\xAB'", - out: &MyMessage{ - Count: Int32(42), - Name: String("\303\277\302\201\xAB"), - }, - }, - - // Bad quoted string - { - in: `inner: < host: "\0" >` + "\n", - err: `line 1.15: invalid quoted string "\0": \0 requires 2 following digits`, - }, - - // Number too large for int64 - { - in: "count: 1 others { key: 123456789012345678901 }", - err: "line 1.23: invalid int64: 123456789012345678901", - }, - - // Number too large for int32 - { - in: "count: 1234567890123", - err: "line 1.7: invalid int32: 1234567890123", - }, - - // Number in hexadecimal - { - in: "count: 0x2beef", - out: &MyMessage{ - Count: Int32(0x2beef), - }, - }, - - // Number in octal - { - in: "count: 024601", - out: &MyMessage{ - Count: Int32(024601), - }, - }, - - // Floating point number with "f" suffix - { - in: "count: 4 others:< weight: 17.0f >", - out: &MyMessage{ - Count: Int32(4), - Others: []*OtherMessage{ - { - Weight: Float32(17), - }, - }, - }, - }, - - // Floating point positive infinity - { - in: "count: 4 bigfloat: inf", - out: &MyMessage{ - Count: Int32(4), - Bigfloat: Float64(math.Inf(1)), - }, - }, - - // Floating point negative infinity - { - in: "count: 4 bigfloat: -inf", - out: &MyMessage{ - Count: Int32(4), - Bigfloat: Float64(math.Inf(-1)), - }, - }, - - // Number too large for float32 - { - in: "others:< weight: 12345678901234567890123456789012345678901234567890 >", - err: "line 1.17: invalid float32: 12345678901234567890123456789012345678901234567890", - }, - - // Number posing as a quoted string - { - in: `inner: < host: 12 >` + "\n", - err: `line 1.15: invalid string: 12`, - }, - - // Quoted string posing as int32 - { - in: `count: "12"`, - err: `line 1.7: invalid int32: "12"`, - }, - - // Quoted string posing a float32 - { - in: `others:< weight: "17.4" >`, - err: `line 1.17: invalid float32: "17.4"`, - }, - - // Enum - { - in: `count:42 bikeshed: BLUE`, - out: &MyMessage{ - Count: Int32(42), - Bikeshed: MyMessage_BLUE.Enum(), - }, - }, - - // Repeated field - { - in: `count:42 pet: "horsey" pet:"bunny"`, - out: &MyMessage{ - Count: Int32(42), - Pet: []string{"horsey", "bunny"}, - }, - }, - - // Repeated field with list notation - { - in: `count:42 pet: ["horsey", "bunny"]`, - out: &MyMessage{ - Count: Int32(42), - Pet: []string{"horsey", "bunny"}, - }, - }, - - // Repeated message with/without colon and <>/{} - { - in: `count:42 others:{} others{} others:<> others:{}`, - out: &MyMessage{ - Count: Int32(42), - Others: []*OtherMessage{ - {}, - {}, - {}, - {}, - }, - }, - }, - - // Missing colon for inner message - { - in: `count:42 inner < host: "cauchy.syd" >`, - out: &MyMessage{ - Count: Int32(42), - Inner: &InnerMessage{ - Host: String("cauchy.syd"), - }, - }, - }, - - // Missing colon for string field - { - in: `name "Dave"`, - err: `line 1.5: expected ':', found "\"Dave\""`, - }, - - // Missing colon for int32 field - { - in: `count 42`, - err: `line 1.6: expected ':', found "42"`, - }, - - // Missing required field - { - in: `name: "Pawel"`, - err: `proto: required field "testdata.MyMessage.count" not set`, - out: &MyMessage{ - Name: String("Pawel"), - }, - }, - - // Repeated non-repeated field - { - in: `name: "Rob" name: "Russ"`, - err: `line 1.12: non-repeated field "name" was repeated`, - }, - - // Group - { - in: `count: 17 SomeGroup { group_field: 12 }`, - out: &MyMessage{ - Count: Int32(17), - Somegroup: &MyMessage_SomeGroup{ - GroupField: Int32(12), - }, - }, - }, - - // Semicolon between fields - { - in: `count:3;name:"Calvin"`, - out: &MyMessage{ - Count: Int32(3), - Name: String("Calvin"), - }, - }, - // Comma between fields - { - in: `count:4,name:"Ezekiel"`, - out: &MyMessage{ - Count: Int32(4), - Name: String("Ezekiel"), - }, - }, - - // Extension - buildExtStructTest(`count: 42 [testdata.Ext.more]:`), - buildExtStructTest(`count: 42 [testdata.Ext.more] {data:"Hello, world!"}`), - buildExtDataTest(`count: 42 [testdata.Ext.text]:"Hello, world!" [testdata.Ext.number]:1729`), - buildExtRepStringTest(`count: 42 [testdata.greeting]:"bula" [testdata.greeting]:"hola"`), - - // Big all-in-one - { - in: "count:42 # Meaning\n" + - `name:"Dave" ` + - `quote:"\"I didn't want to go.\"" ` + - `pet:"bunny" ` + - `pet:"kitty" ` + - `pet:"horsey" ` + - `inner:<` + - ` host:"footrest.syd" ` + - ` port:7001 ` + - ` connected:true ` + - `> ` + - `others:<` + - ` key:3735928559 ` + - ` value:"\x01A\a\f" ` + - `> ` + - `others:<` + - " weight:58.9 # Atomic weight of Co\n" + - ` inner:<` + - ` host:"lesha.mtv" ` + - ` port:8002 ` + - ` >` + - `>`, - out: &MyMessage{ - Count: Int32(42), - Name: String("Dave"), - Quote: String(`"I didn't want to go."`), - Pet: []string{"bunny", "kitty", "horsey"}, - Inner: &InnerMessage{ - Host: String("footrest.syd"), - Port: Int32(7001), - Connected: Bool(true), - }, - Others: []*OtherMessage{ - { - Key: Int64(3735928559), - Value: []byte{0x1, 'A', '\a', '\f'}, - }, - { - Weight: Float32(58.9), - Inner: &InnerMessage{ - Host: String("lesha.mtv"), - Port: Int32(8002), - }, - }, - }, - }, - }, -} - -func TestUnmarshalText(t *testing.T) { - for i, test := range unMarshalTextTests { - pb := new(MyMessage) - err := UnmarshalText(test.in, pb) - if test.err == "" { - // We don't expect failure. - if err != nil { - t.Errorf("Test %d: Unexpected error: %v", i, err) - } else if !reflect.DeepEqual(pb, test.out) { - t.Errorf("Test %d: Incorrect populated \nHave: %v\nWant: %v", - i, pb, test.out) - } - } else { - // We do expect failure. - if err == nil { - t.Errorf("Test %d: Didn't get expected error: %v", i, test.err) - } else if err.Error() != test.err { - t.Errorf("Test %d: Incorrect error.\nHave: %v\nWant: %v", - i, err.Error(), test.err) - } else if _, ok := err.(*RequiredNotSetError); ok && test.out != nil && !reflect.DeepEqual(pb, test.out) { - t.Errorf("Test %d: Incorrect populated \nHave: %v\nWant: %v", - i, pb, test.out) - } - } - } -} - -func TestUnmarshalTextCustomMessage(t *testing.T) { - msg := &textMessage{} - if err := UnmarshalText("custom", msg); err != nil { - t.Errorf("Unexpected error from custom unmarshal: %v", err) - } - if UnmarshalText("not custom", msg) == nil { - t.Errorf("Didn't get expected error from custom unmarshal") - } -} - -// Regression test; this caused a panic. -func TestRepeatedEnum(t *testing.T) { - pb := new(RepeatedEnum) - if err := UnmarshalText("color: RED", pb); err != nil { - t.Fatal(err) - } - exp := &RepeatedEnum{ - Color: []RepeatedEnum_Color{RepeatedEnum_RED}, - } - if !Equal(pb, exp) { - t.Errorf("Incorrect populated \nHave: %v\nWant: %v", pb, exp) - } -} - -func TestProto3TextParsing(t *testing.T) { - m := new(proto3pb.Message) - const in = `name: "Wallace" true_scotsman: true` - want := &proto3pb.Message{ - Name: "Wallace", - TrueScotsman: true, - } - if err := UnmarshalText(in, m); err != nil { - t.Fatal(err) - } - if !Equal(m, want) { - t.Errorf("\n got %v\nwant %v", m, want) - } -} - -func TestMapParsing(t *testing.T) { - m := new(MessageWithMap) - const in = `name_mapping: name_mapping:` + - `msg_mapping:,>` + // separating commas are okay - `msg_mapping>` + // no colon after "value" - `byte_mapping:` - want := &MessageWithMap{ - NameMapping: map[int32]string{ - 1: "Beatles", - 1234: "Feist", - }, - MsgMapping: map[int64]*FloatingPoint{ - -4: {F: Float64(2.0)}, - -2: {F: Float64(4.0)}, - }, - ByteMapping: map[bool][]byte{ - true: []byte("so be it"), - }, - } - if err := UnmarshalText(in, m); err != nil { - t.Fatal(err) - } - if !Equal(m, want) { - t.Errorf("\n got %v\nwant %v", m, want) - } -} - -func TestOneofParsing(t *testing.T) { - const in = `name:"Shrek"` - m := new(Communique) - want := &Communique{Union: &Communique_Name{"Shrek"}} - if err := UnmarshalText(in, m); err != nil { - t.Fatal(err) - } - if !Equal(m, want) { - t.Errorf("\n got %v\nwant %v", m, want) - } -} - -var benchInput string - -func init() { - benchInput = "count: 4\n" - for i := 0; i < 1000; i++ { - benchInput += "pet: \"fido\"\n" - } - - // Check it is valid input. - pb := new(MyMessage) - err := UnmarshalText(benchInput, pb) - if err != nil { - panic("Bad benchmark input: " + err.Error()) - } -} - -func BenchmarkUnmarshalText(b *testing.B) { - pb := new(MyMessage) - for i := 0; i < b.N; i++ { - UnmarshalText(benchInput, pb) - } - b.SetBytes(int64(len(benchInput))) -} diff --git a/vendor/github.com/golang/protobuf/proto/text_test.go b/vendor/github.com/golang/protobuf/proto/text_test.go deleted file mode 100644 index 3eabacac..00000000 --- a/vendor/github.com/golang/protobuf/proto/text_test.go +++ /dev/null @@ -1,474 +0,0 @@ -// Go support for Protocol Buffers - Google's data interchange format -// -// Copyright 2010 The Go Authors. All rights reserved. -// https://github.com/golang/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package proto_test - -import ( - "bytes" - "errors" - "io/ioutil" - "math" - "strings" - "testing" - - "github.com/golang/protobuf/proto" - - proto3pb "github.com/golang/protobuf/proto/proto3_proto" - pb "github.com/golang/protobuf/proto/testdata" -) - -// textMessage implements the methods that allow it to marshal and unmarshal -// itself as text. -type textMessage struct { -} - -func (*textMessage) MarshalText() ([]byte, error) { - return []byte("custom"), nil -} - -func (*textMessage) UnmarshalText(bytes []byte) error { - if string(bytes) != "custom" { - return errors.New("expected 'custom'") - } - return nil -} - -func (*textMessage) Reset() {} -func (*textMessage) String() string { return "" } -func (*textMessage) ProtoMessage() {} - -func newTestMessage() *pb.MyMessage { - msg := &pb.MyMessage{ - Count: proto.Int32(42), - Name: proto.String("Dave"), - Quote: proto.String(`"I didn't want to go."`), - Pet: []string{"bunny", "kitty", "horsey"}, - Inner: &pb.InnerMessage{ - Host: proto.String("footrest.syd"), - Port: proto.Int32(7001), - Connected: proto.Bool(true), - }, - Others: []*pb.OtherMessage{ - { - Key: proto.Int64(0xdeadbeef), - Value: []byte{1, 65, 7, 12}, - }, - { - Weight: proto.Float32(6.022), - Inner: &pb.InnerMessage{ - Host: proto.String("lesha.mtv"), - Port: proto.Int32(8002), - }, - }, - }, - Bikeshed: pb.MyMessage_BLUE.Enum(), - Somegroup: &pb.MyMessage_SomeGroup{ - GroupField: proto.Int32(8), - }, - // One normally wouldn't do this. - // This is an undeclared tag 13, as a varint (wire type 0) with value 4. - XXX_unrecognized: []byte{13<<3 | 0, 4}, - } - ext := &pb.Ext{ - Data: proto.String("Big gobs for big rats"), - } - if err := proto.SetExtension(msg, pb.E_Ext_More, ext); err != nil { - panic(err) - } - greetings := []string{"adg", "easy", "cow"} - if err := proto.SetExtension(msg, pb.E_Greeting, greetings); err != nil { - panic(err) - } - - // Add an unknown extension. We marshal a pb.Ext, and fake the ID. - b, err := proto.Marshal(&pb.Ext{Data: proto.String("3G skiing")}) - if err != nil { - panic(err) - } - b = append(proto.EncodeVarint(201<<3|proto.WireBytes), b...) - proto.SetRawExtension(msg, 201, b) - - // Extensions can be plain fields, too, so let's test that. - b = append(proto.EncodeVarint(202<<3|proto.WireVarint), 19) - proto.SetRawExtension(msg, 202, b) - - return msg -} - -const text = `count: 42 -name: "Dave" -quote: "\"I didn't want to go.\"" -pet: "bunny" -pet: "kitty" -pet: "horsey" -inner: < - host: "footrest.syd" - port: 7001 - connected: true -> -others: < - key: 3735928559 - value: "\001A\007\014" -> -others: < - weight: 6.022 - inner: < - host: "lesha.mtv" - port: 8002 - > -> -bikeshed: BLUE -SomeGroup { - group_field: 8 -} -/* 2 unknown bytes */ -13: 4 -[testdata.Ext.more]: < - data: "Big gobs for big rats" -> -[testdata.greeting]: "adg" -[testdata.greeting]: "easy" -[testdata.greeting]: "cow" -/* 13 unknown bytes */ -201: "\t3G skiing" -/* 3 unknown bytes */ -202: 19 -` - -func TestMarshalText(t *testing.T) { - buf := new(bytes.Buffer) - if err := proto.MarshalText(buf, newTestMessage()); err != nil { - t.Fatalf("proto.MarshalText: %v", err) - } - s := buf.String() - if s != text { - t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, text) - } -} - -func TestMarshalTextCustomMessage(t *testing.T) { - buf := new(bytes.Buffer) - if err := proto.MarshalText(buf, &textMessage{}); err != nil { - t.Fatalf("proto.MarshalText: %v", err) - } - s := buf.String() - if s != "custom" { - t.Errorf("Got %q, expected %q", s, "custom") - } -} -func TestMarshalTextNil(t *testing.T) { - want := "" - tests := []proto.Message{nil, (*pb.MyMessage)(nil)} - for i, test := range tests { - buf := new(bytes.Buffer) - if err := proto.MarshalText(buf, test); err != nil { - t.Fatal(err) - } - if got := buf.String(); got != want { - t.Errorf("%d: got %q want %q", i, got, want) - } - } -} - -func TestMarshalTextUnknownEnum(t *testing.T) { - // The Color enum only specifies values 0-2. - m := &pb.MyMessage{Bikeshed: pb.MyMessage_Color(3).Enum()} - got := m.String() - const want = `bikeshed:3 ` - if got != want { - t.Errorf("\n got %q\nwant %q", got, want) - } -} - -func TestTextOneof(t *testing.T) { - tests := []struct { - m proto.Message - want string - }{ - // zero message - {&pb.Communique{}, ``}, - // scalar field - {&pb.Communique{Union: &pb.Communique_Number{4}}, `number:4`}, - // message field - {&pb.Communique{Union: &pb.Communique_Msg{ - &pb.Strings{StringField: proto.String("why hello!")}, - }}, `msg:`}, - // bad oneof (should not panic) - {&pb.Communique{Union: &pb.Communique_Msg{nil}}, `msg:/* nil */`}, - } - for _, test := range tests { - got := strings.TrimSpace(test.m.String()) - if got != test.want { - t.Errorf("\n got %s\nwant %s", got, test.want) - } - } -} - -func BenchmarkMarshalTextBuffered(b *testing.B) { - buf := new(bytes.Buffer) - m := newTestMessage() - for i := 0; i < b.N; i++ { - buf.Reset() - proto.MarshalText(buf, m) - } -} - -func BenchmarkMarshalTextUnbuffered(b *testing.B) { - w := ioutil.Discard - m := newTestMessage() - for i := 0; i < b.N; i++ { - proto.MarshalText(w, m) - } -} - -func compact(src string) string { - // s/[ \n]+/ /g; s/ $//; - dst := make([]byte, len(src)) - space, comment := false, false - j := 0 - for i := 0; i < len(src); i++ { - if strings.HasPrefix(src[i:], "/*") { - comment = true - i++ - continue - } - if comment && strings.HasPrefix(src[i:], "*/") { - comment = false - i++ - continue - } - if comment { - continue - } - c := src[i] - if c == ' ' || c == '\n' { - space = true - continue - } - if j > 0 && (dst[j-1] == ':' || dst[j-1] == '<' || dst[j-1] == '{') { - space = false - } - if c == '{' { - space = false - } - if space { - dst[j] = ' ' - j++ - space = false - } - dst[j] = c - j++ - } - if space { - dst[j] = ' ' - j++ - } - return string(dst[0:j]) -} - -var compactText = compact(text) - -func TestCompactText(t *testing.T) { - s := proto.CompactTextString(newTestMessage()) - if s != compactText { - t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v\n===\n", s, compactText) - } -} - -func TestStringEscaping(t *testing.T) { - testCases := []struct { - in *pb.Strings - out string - }{ - { - // Test data from C++ test (TextFormatTest.StringEscape). - // Single divergence: we don't escape apostrophes. - &pb.Strings{StringField: proto.String("\"A string with ' characters \n and \r newlines and \t tabs and \001 slashes \\ and multiple spaces")}, - "string_field: \"\\\"A string with ' characters \\n and \\r newlines and \\t tabs and \\001 slashes \\\\ and multiple spaces\"\n", - }, - { - // Test data from the same C++ test. - &pb.Strings{StringField: proto.String("\350\260\267\346\255\214")}, - "string_field: \"\\350\\260\\267\\346\\255\\214\"\n", - }, - { - // Some UTF-8. - &pb.Strings{StringField: proto.String("\x00\x01\xff\x81")}, - `string_field: "\000\001\377\201"` + "\n", - }, - } - - for i, tc := range testCases { - var buf bytes.Buffer - if err := proto.MarshalText(&buf, tc.in); err != nil { - t.Errorf("proto.MarsalText: %v", err) - continue - } - s := buf.String() - if s != tc.out { - t.Errorf("#%d: Got:\n%s\nExpected:\n%s\n", i, s, tc.out) - continue - } - - // Check round-trip. - pb := new(pb.Strings) - if err := proto.UnmarshalText(s, pb); err != nil { - t.Errorf("#%d: UnmarshalText: %v", i, err) - continue - } - if !proto.Equal(pb, tc.in) { - t.Errorf("#%d: Round-trip failed:\nstart: %v\n end: %v", i, tc.in, pb) - } - } -} - -// A limitedWriter accepts some output before it fails. -// This is a proxy for something like a nearly-full or imminently-failing disk, -// or a network connection that is about to die. -type limitedWriter struct { - b bytes.Buffer - limit int -} - -var outOfSpace = errors.New("proto: insufficient space") - -func (w *limitedWriter) Write(p []byte) (n int, err error) { - var avail = w.limit - w.b.Len() - if avail <= 0 { - return 0, outOfSpace - } - if len(p) <= avail { - return w.b.Write(p) - } - n, _ = w.b.Write(p[:avail]) - return n, outOfSpace -} - -func TestMarshalTextFailing(t *testing.T) { - // Try lots of different sizes to exercise more error code-paths. - for lim := 0; lim < len(text); lim++ { - buf := new(limitedWriter) - buf.limit = lim - err := proto.MarshalText(buf, newTestMessage()) - // We expect a certain error, but also some partial results in the buffer. - if err != outOfSpace { - t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", err, outOfSpace) - } - s := buf.b.String() - x := text[:buf.limit] - if s != x { - t.Errorf("Got:\n===\n%v===\nExpected:\n===\n%v===\n", s, x) - } - } -} - -func TestFloats(t *testing.T) { - tests := []struct { - f float64 - want string - }{ - {0, "0"}, - {4.7, "4.7"}, - {math.Inf(1), "inf"}, - {math.Inf(-1), "-inf"}, - {math.NaN(), "nan"}, - } - for _, test := range tests { - msg := &pb.FloatingPoint{F: &test.f} - got := strings.TrimSpace(msg.String()) - want := `f:` + test.want - if got != want { - t.Errorf("f=%f: got %q, want %q", test.f, got, want) - } - } -} - -func TestRepeatedNilText(t *testing.T) { - m := &pb.MessageList{ - Message: []*pb.MessageList_Message{ - nil, - &pb.MessageList_Message{ - Name: proto.String("Horse"), - }, - nil, - }, - } - want := `Message -Message { - name: "Horse" -} -Message -` - if s := proto.MarshalTextString(m); s != want { - t.Errorf(" got: %s\nwant: %s", s, want) - } -} - -func TestProto3Text(t *testing.T) { - tests := []struct { - m proto.Message - want string - }{ - // zero message - {&proto3pb.Message{}, ``}, - // zero message except for an empty byte slice - {&proto3pb.Message{Data: []byte{}}, ``}, - // trivial case - {&proto3pb.Message{Name: "Rob", HeightInCm: 175}, `name:"Rob" height_in_cm:175`}, - // empty map - {&pb.MessageWithMap{}, ``}, - // non-empty map; map format is the same as a repeated struct, - // and they are sorted by key (numerically for numeric keys). - { - &pb.MessageWithMap{NameMapping: map[int32]string{ - -1: "Negatory", - 7: "Lucky", - 1234: "Feist", - 6345789: "Otis", - }}, - `name_mapping: ` + - `name_mapping: ` + - `name_mapping: ` + - `name_mapping:`, - }, - // map with nil value; not well-defined, but we shouldn't crash - { - &pb.MessageWithMap{MsgMapping: map[int64]*pb.FloatingPoint{7: nil}}, - `msg_mapping:`, - }, - } - for _, test := range tests { - got := strings.TrimSpace(test.m.String()) - if got != test.want { - t.Errorf("\n got %s\nwant %s", got, test.want) - } - } -} diff --git a/vendor/github.com/golang/snappy/AUTHORS b/vendor/github.com/golang/snappy/AUTHORS deleted file mode 100644 index bcfa1952..00000000 --- a/vendor/github.com/golang/snappy/AUTHORS +++ /dev/null @@ -1,15 +0,0 @@ -# This is the official list of Snappy-Go authors for copyright purposes. -# This file is distinct from the CONTRIBUTORS files. -# See the latter for an explanation. - -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# Please keep the list sorted. - -Damian Gryski -Google Inc. -Jan Mercl <0xjnml@gmail.com> -Rodolfo Carvalho -Sebastien Binet diff --git a/vendor/github.com/golang/snappy/CONTRIBUTORS b/vendor/github.com/golang/snappy/CONTRIBUTORS deleted file mode 100644 index 931ae316..00000000 --- a/vendor/github.com/golang/snappy/CONTRIBUTORS +++ /dev/null @@ -1,37 +0,0 @@ -# This is the official list of people who can contribute -# (and typically have contributed) code to the Snappy-Go repository. -# The AUTHORS file lists the copyright holders; this file -# lists people. For example, Google employees are listed here -# but not in AUTHORS, because Google holds the copyright. -# -# The submission process automatically checks to make sure -# that people submitting code are listed in this file (by email address). -# -# Names should be added to this file only after verifying that -# the individual or the individual's organization has agreed to -# the appropriate Contributor License Agreement, found here: -# -# http://code.google.com/legal/individual-cla-v1.0.html -# http://code.google.com/legal/corporate-cla-v1.0.html -# -# The agreement for individuals can be filled out on the web. -# -# When adding J Random Contributor's name to this file, -# either J's name or J's organization's name should be -# added to the AUTHORS file, depending on whether the -# individual or corporate CLA was used. - -# Names should be added to this file like so: -# Name - -# Please keep the list sorted. - -Damian Gryski -Jan Mercl <0xjnml@gmail.com> -Kai Backman -Marc-Antoine Ruel -Nigel Tao -Rob Pike -Rodolfo Carvalho -Russ Cox -Sebastien Binet diff --git a/vendor/github.com/golang/snappy/README b/vendor/github.com/golang/snappy/README deleted file mode 100644 index 5074bbab..00000000 --- a/vendor/github.com/golang/snappy/README +++ /dev/null @@ -1,7 +0,0 @@ -The Snappy compression format in the Go programming language. - -To download and install from source: -$ go get github.com/golang/snappy - -Unless otherwise noted, the Snappy-Go source files are distributed -under the BSD-style license found in the LICENSE file. diff --git a/vendor/github.com/golang/snappy/cmd/snappytool/main.cpp b/vendor/github.com/golang/snappy/cmd/snappytool/main.cpp deleted file mode 100644 index db28a899..00000000 --- a/vendor/github.com/golang/snappy/cmd/snappytool/main.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* -To build the snappytool binary: -g++ main.cpp /usr/lib/libsnappy.a -o snappytool -*/ - -#include -#include -#include -#include - -#include "snappy.h" - -#define N 1000000 - -char dst[N]; -char src[N]; - -int main(int argc, char** argv) { - // Parse args. - if (argc != 2) { - fprintf(stderr, "exactly one of -d or -e must be given\n"); - return 1; - } - bool decode = strcmp(argv[1], "-d") == 0; - bool encode = strcmp(argv[1], "-e") == 0; - if (decode == encode) { - fprintf(stderr, "exactly one of -d or -e must be given\n"); - return 1; - } - - // Read all of stdin into src[:s]. - size_t s = 0; - while (1) { - if (s == N) { - fprintf(stderr, "input too large\n"); - return 1; - } - ssize_t n = read(0, src+s, N-s); - if (n == 0) { - break; - } - if (n < 0) { - fprintf(stderr, "read error: %s\n", strerror(errno)); - // TODO: handle EAGAIN, EINTR? - return 1; - } - s += n; - } - - // Encode or decode src[:s] to dst[:d], and write to stdout. - size_t d = 0; - if (encode) { - if (N < snappy::MaxCompressedLength(s)) { - fprintf(stderr, "input too large after encoding\n"); - return 1; - } - snappy::RawCompress(src, s, dst, &d); - } else { - if (!snappy::GetUncompressedLength(src, s, &d)) { - fprintf(stderr, "could not get uncompressed length\n"); - return 1; - } - if (N < d) { - fprintf(stderr, "input too large after decoding\n"); - return 1; - } - if (!snappy::RawUncompress(src, s, dst)) { - fprintf(stderr, "input was not valid Snappy-compressed data\n"); - return 1; - } - } - write(1, dst, d); - return 0; -} diff --git a/vendor/github.com/golang/snappy/decode.go b/vendor/github.com/golang/snappy/decode.go deleted file mode 100644 index 7be590ce..00000000 --- a/vendor/github.com/golang/snappy/decode.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -import ( - "encoding/binary" - "errors" - "io" -) - -var ( - // ErrCorrupt reports that the input is invalid. - ErrCorrupt = errors.New("snappy: corrupt input") - // ErrTooLarge reports that the uncompressed length is too large. - ErrTooLarge = errors.New("snappy: decoded block is too large") - // ErrUnsupported reports that the input isn't supported. - ErrUnsupported = errors.New("snappy: unsupported input") - - errUnsupportedCopy4Tag = errors.New("snappy: unsupported COPY_4 tag") - errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") -) - -// DecodedLen returns the length of the decoded block. -func DecodedLen(src []byte) (int, error) { - v, _, err := decodedLen(src) - return v, err -} - -// decodedLen returns the length of the decoded block and the number of bytes -// that the length header occupied. -func decodedLen(src []byte) (blockLen, headerLen int, err error) { - v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { - return 0, 0, ErrCorrupt - } - - const wordSize = 32 << (^uint(0) >> 32 & 1) - if wordSize == 32 && v > 0x7fffffff { - return 0, 0, ErrTooLarge - } - return int(v), n, nil -} - -const ( - decodeErrCodeCorrupt = 1 - decodeErrCodeUnsupportedLiteralLength = 2 - decodeErrCodeUnsupportedCopy4Tag = 3 -) - -// Decode returns the decoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire decoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -func Decode(dst, src []byte) ([]byte, error) { - dLen, s, err := decodedLen(src) - if err != nil { - return nil, err - } - if dLen <= len(dst) { - dst = dst[:dLen] - } else { - dst = make([]byte, dLen) - } - switch decode(dst, src[s:]) { - case 0: - return dst, nil - case decodeErrCodeUnsupportedLiteralLength: - return nil, errUnsupportedLiteralLength - case decodeErrCodeUnsupportedCopy4Tag: - return nil, errUnsupportedCopy4Tag - } - return nil, ErrCorrupt -} - -// NewReader returns a new Reader that decompresses from r, using the framing -// format described at -// https://github.com/google/snappy/blob/master/framing_format.txt -func NewReader(r io.Reader) *Reader { - return &Reader{ - r: r, - decoded: make([]byte, maxBlockSize), - buf: make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize), - } -} - -// Reader is an io.Reader that can read Snappy-compressed bytes. -type Reader struct { - r io.Reader - err error - decoded []byte - buf []byte - // decoded[i:j] contains decoded bytes that have not yet been passed on. - i, j int - readHeader bool -} - -// Reset discards any buffered data, resets all state, and switches the Snappy -// reader to read from r. This permits reusing a Reader rather than allocating -// a new one. -func (r *Reader) Reset(reader io.Reader) { - r.r = reader - r.err = nil - r.i = 0 - r.j = 0 - r.readHeader = false -} - -func (r *Reader) readFull(p []byte) (ok bool) { - if _, r.err = io.ReadFull(r.r, p); r.err != nil { - if r.err == io.ErrUnexpectedEOF { - r.err = ErrCorrupt - } - return false - } - return true -} - -// Read satisfies the io.Reader interface. -func (r *Reader) Read(p []byte) (int, error) { - if r.err != nil { - return 0, r.err - } - for { - if r.i < r.j { - n := copy(p, r.decoded[r.i:r.j]) - r.i += n - return n, nil - } - if !r.readFull(r.buf[:4]) { - return 0, r.err - } - chunkType := r.buf[0] - if !r.readHeader { - if chunkType != chunkTypeStreamIdentifier { - r.err = ErrCorrupt - return 0, r.err - } - r.readHeader = true - } - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 - if chunkLen > len(r.buf) { - r.err = ErrUnsupported - return 0, r.err - } - - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - // Section 4.2. Compressed data (chunk type 0x00). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - buf := r.buf[:chunkLen] - if !r.readFull(buf) { - return 0, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - buf = buf[checksumSize:] - - n, err := DecodedLen(buf) - if err != nil { - r.err = err - return 0, r.err - } - if n > len(r.decoded) { - r.err = ErrCorrupt - return 0, r.err - } - if _, err := Decode(r.decoded, buf); err != nil { - r.err = err - return 0, r.err - } - if crc(r.decoded[:n]) != checksum { - r.err = ErrCorrupt - return 0, r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeUncompressedData: - // Section 4.3. Uncompressed data (chunk type 0x01). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - buf := r.buf[:checksumSize] - if !r.readFull(buf) { - return 0, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - // Read directly into r.decoded instead of via r.buf. - n := chunkLen - checksumSize - if !r.readFull(r.decoded[:n]) { - return 0, r.err - } - if crc(r.decoded[:n]) != checksum { - r.err = ErrCorrupt - return 0, r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeStreamIdentifier: - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(magicBody) { - r.err = ErrCorrupt - return 0, r.err - } - if !r.readFull(r.buf[:len(magicBody)]) { - return 0, r.err - } - for i := 0; i < len(magicBody); i++ { - if r.buf[i] != magicBody[i] { - r.err = ErrCorrupt - return 0, r.err - } - } - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - r.err = ErrUnsupported - return 0, r.err - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - if !r.readFull(r.buf[:chunkLen]) { - return 0, r.err - } - } -} diff --git a/vendor/github.com/golang/snappy/decode_amd64.go b/vendor/github.com/golang/snappy/decode_amd64.go deleted file mode 100644 index 32bce470..00000000 --- a/vendor/github.com/golang/snappy/decode_amd64.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -// decode has the same semantics as in decode_other.go. -// -//go:noescape -func decode(dst, src []byte) int diff --git a/vendor/github.com/golang/snappy/decode_amd64.s b/vendor/github.com/golang/snappy/decode_amd64.s deleted file mode 100644 index 1486aba6..00000000 --- a/vendor/github.com/golang/snappy/decode_amd64.s +++ /dev/null @@ -1,472 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "textflag.h" - -// func decode(dst, src []byte) int -// -// The asm code generally follows the pure Go code in decode_other.go, except -// where marked with a "!!!". -// -// All local variables fit into registers. The non-zero stack size is only to -// spill registers and push args when issuing a CALL. The register allocation: -// - AX scratch -// - BX scratch -// - CX length or x -// - DX offset -// - SI &src[s] -// - DI &dst[d] -// + R8 dst_base -// + R9 dst_len -// + R10 dst_base + dst_len -// + R11 src_base -// + R12 src_len -// + R13 src_base + src_len -// - R14 used by doCopy -// - R15 used by doCopy -// -// The registers R8-R13 (marked with a "+") are set at the start of the -// function, and after a CALL returns, and are not otherwise modified. -// -// The d variable is implicitly DI - R8, and len(dst)-d is R10 - DI. -// The s variable is implicitly SI - R11, and len(src)-s is R13 - SI. -TEXT ·decode(SB), NOSPLIT, $48-56 - // Initialize SI, DI and R8-R13. - MOVQ dst_base+0(FP), R8 - MOVQ dst_len+8(FP), R9 - MOVQ R8, DI - MOVQ R8, R10 - ADDQ R9, R10 - MOVQ src_base+24(FP), R11 - MOVQ src_len+32(FP), R12 - MOVQ R11, SI - MOVQ R11, R13 - ADDQ R12, R13 - -loop: - // for s < len(src) - CMPQ SI, R13 - JEQ end - - // CX = uint32(src[s]) - // - // switch src[s] & 0x03 - MOVBLZX (SI), CX - MOVL CX, BX - ANDL $3, BX - CMPL BX, $1 - JAE tagCopy - - // ---------------------------------------- - // The code below handles literal tags. - - // case tagLiteral: - // x := uint32(src[s] >> 2) - // switch - SHRL $2, CX - CMPL CX, $60 - JAE tagLit60Plus - - // case x < 60: - // s++ - INCQ SI - -doLit: - // This is the end of the inner "switch", when we have a literal tag. - // - // We assume that CX == x and x fits in a uint32, where x is the variable - // used in the pure Go decode_other.go code. - - // length = int(x) + 1 - // - // Unlike the pure Go code, we don't need to check if length <= 0 because - // CX can hold 64 bits, so the increment cannot overflow. - INCQ CX - - // Prepare to check if copying length bytes will run past the end of dst or - // src. - // - // AX = len(dst) - d - // BX = len(src) - s - MOVQ R10, AX - SUBQ DI, AX - MOVQ R13, BX - SUBQ SI, BX - - // !!! Try a faster technique for short (16 or fewer bytes) copies. - // - // if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { - // goto callMemmove // Fall back on calling runtime·memmove. - // } - // - // The C++ snappy code calls this TryFastAppend. It also checks len(src)-s - // against 21 instead of 16, because it cannot assume that all of its input - // is contiguous in memory and so it needs to leave enough source bytes to - // read the next tag without refilling buffers, but Go's Decode assumes - // contiguousness (the src argument is a []byte). - CMPQ CX, $16 - JGT callMemmove - CMPQ AX, $16 - JLT callMemmove - CMPQ BX, $16 - JLT callMemmove - - // !!! Implement the copy from src to dst as two 8-byte loads and stores. - // (Decode's documentation says that dst and src must not overlap.) - // - // This always copies 16 bytes, instead of only length bytes, but that's - // OK. If the input is a valid Snappy encoding then subsequent iterations - // will fix up the overrun. Otherwise, Decode returns a nil []byte (and a - // non-nil error), so the overrun will be ignored. - // - // Note that on amd64, it is legal and cheap to issue unaligned 8-byte - // loads and stores. This technique probably wouldn't be as effective on - // architectures that are fussier about alignment. - MOVQ 0(SI), AX - MOVQ AX, 0(DI) - MOVQ 8(SI), BX - MOVQ BX, 8(DI) - - // d += length - // s += length - ADDQ CX, DI - ADDQ CX, SI - JMP loop - -callMemmove: - // if length > len(dst)-d || length > len(src)-s { etc } - CMPQ CX, AX - JGT errCorrupt - CMPQ CX, BX - JGT errCorrupt - - // copy(dst[d:], src[s:s+length]) - // - // This means calling runtime·memmove(&dst[d], &src[s], length), so we push - // DI, SI and CX as arguments. Coincidentally, we also need to spill those - // three registers to the stack, to save local variables across the CALL. - MOVQ DI, 0(SP) - MOVQ SI, 8(SP) - MOVQ CX, 16(SP) - MOVQ DI, 24(SP) - MOVQ SI, 32(SP) - MOVQ CX, 40(SP) - CALL runtime·memmove(SB) - - // Restore local variables: unspill registers from the stack and - // re-calculate R8-R13. - MOVQ 24(SP), DI - MOVQ 32(SP), SI - MOVQ 40(SP), CX - MOVQ dst_base+0(FP), R8 - MOVQ dst_len+8(FP), R9 - MOVQ R8, R10 - ADDQ R9, R10 - MOVQ src_base+24(FP), R11 - MOVQ src_len+32(FP), R12 - MOVQ R11, R13 - ADDQ R12, R13 - - // d += length - // s += length - ADDQ CX, DI - ADDQ CX, SI - JMP loop - -tagLit60Plus: - // !!! This fragment does the - // - // s += x - 58; if uint(s) > uint(len(src)) { etc } - // - // checks. In the asm version, we code it once instead of once per switch case. - ADDQ CX, SI - SUBQ $58, SI - MOVQ SI, BX - SUBQ R11, BX - CMPQ BX, R12 - JA errCorrupt - - // case x == 60: - CMPL CX, $61 - JEQ tagLit61 - JA tagLit62Plus - - // x = uint32(src[s-1]) - MOVBLZX -1(SI), CX - JMP doLit - -tagLit61: - // case x == 61: - // x = uint32(src[s-2]) | uint32(src[s-1])<<8 - MOVWLZX -2(SI), CX - JMP doLit - -tagLit62Plus: - CMPL CX, $62 - JA tagLit63 - - // case x == 62: - // x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - MOVWLZX -3(SI), CX - MOVBLZX -1(SI), BX - SHLL $16, BX - ORL BX, CX - JMP doLit - -tagLit63: - // case x == 63: - // x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - MOVL -4(SI), CX - JMP doLit - -// The code above handles literal tags. -// ---------------------------------------- -// The code below handles copy tags. - -tagCopy2: - // case tagCopy2: - // s += 3 - ADDQ $3, SI - - // if uint(s) > uint(len(src)) { etc } - MOVQ SI, BX - SUBQ R11, BX - CMPQ BX, R12 - JA errCorrupt - - // length = 1 + int(src[s-3])>>2 - SHRQ $2, CX - INCQ CX - - // offset = int(src[s-2]) | int(src[s-1])<<8 - MOVWQZX -2(SI), DX - JMP doCopy - -tagCopy: - // We have a copy tag. We assume that: - // - BX == src[s] & 0x03 - // - CX == src[s] - CMPQ BX, $2 - JEQ tagCopy2 - JA errUC4T - - // case tagCopy1: - // s += 2 - ADDQ $2, SI - - // if uint(s) > uint(len(src)) { etc } - MOVQ SI, BX - SUBQ R11, BX - CMPQ BX, R12 - JA errCorrupt - - // offset = int(src[s-2])&0xe0<<3 | int(src[s-1]) - MOVQ CX, DX - ANDQ $0xe0, DX - SHLQ $3, DX - MOVBQZX -1(SI), BX - ORQ BX, DX - - // length = 4 + int(src[s-2])>>2&0x7 - SHRQ $2, CX - ANDQ $7, CX - ADDQ $4, CX - -doCopy: - // This is the end of the outer "switch", when we have a copy tag. - // - // We assume that: - // - CX == length && CX > 0 - // - DX == offset - - // if offset <= 0 { etc } - CMPQ DX, $0 - JLE errCorrupt - - // if d < offset { etc } - MOVQ DI, BX - SUBQ R8, BX - CMPQ BX, DX - JLT errCorrupt - - // if length > len(dst)-d { etc } - MOVQ R10, BX - SUBQ DI, BX - CMPQ CX, BX - JGT errCorrupt - - // forwardCopy(dst[d:d+length], dst[d-offset:]); d += length - // - // Set: - // - R14 = len(dst)-d - // - R15 = &dst[d-offset] - MOVQ R10, R14 - SUBQ DI, R14 - MOVQ DI, R15 - SUBQ DX, R15 - - // !!! Try a faster technique for short (16 or fewer bytes) forward copies. - // - // First, try using two 8-byte load/stores, similar to the doLit technique - // above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is - // still OK if offset >= 8. - // - // if length > 16 || offset < 8 || len(dst)-d < 16 { - // goto slowForwardCopy - // } - // copy 16 bytes - // d += length - CMPQ CX, $16 - JGT slowForwardCopy - CMPQ DX, $8 - JLT slowForwardCopy - CMPQ R14, $16 - JLT slowForwardCopy - MOVQ 0(R15), AX - MOVQ AX, 0(DI) - MOVQ 8(R15), BX - MOVQ BX, 8(DI) - ADDQ CX, DI - JMP loop - -slowForwardCopy: - // !!! If the forward copy is longer than 16 bytes, or if offset < 8, we - // can still try 8-byte load stores, provided we can overrun up to 10 extra - // bytes. As above, the overrun will be fixed up by subsequent iterations - // of the outermost loop. - // - // The C++ snappy code calls this technique IncrementalCopyFastPath. Its - // commentary says: - // - // ---- - // - // The main part of this loop is a simple copy of eight bytes at a time - // until we've copied (at least) the requested amount of bytes. However, - // if d and d-offset are less than eight bytes apart (indicating a - // repeating pattern of length < 8), we first need to expand the pattern in - // order to get the correct results. For instance, if the buffer looks like - // this, with the eight-byte and patterns marked as - // intervals: - // - // abxxxxxxxxxxxx - // [------] d-offset - // [------] d - // - // a single eight-byte copy from to will repeat the pattern - // once, after which we can move two bytes without moving : - // - // ababxxxxxxxxxx - // [------] d-offset - // [------] d - // - // and repeat the exercise until the two no longer overlap. - // - // This allows us to do very well in the special case of one single byte - // repeated many times, without taking a big hit for more general cases. - // - // The worst case of extra writing past the end of the match occurs when - // offset == 1 and length == 1; the last copy will read from byte positions - // [0..7] and write to [4..11], whereas it was only supposed to write to - // position 1. Thus, ten excess bytes. - // - // ---- - // - // That "10 byte overrun" worst case is confirmed by Go's - // TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy - // and finishSlowForwardCopy algorithm. - // - // if length > len(dst)-d-10 { - // goto verySlowForwardCopy - // } - SUBQ $10, R14 - CMPQ CX, R14 - JGT verySlowForwardCopy - -makeOffsetAtLeast8: - // !!! As above, expand the pattern so that offset >= 8 and we can use - // 8-byte load/stores. - // - // for offset < 8 { - // copy 8 bytes from dst[d-offset:] to dst[d:] - // length -= offset - // d += offset - // offset += offset - // // The two previous lines together means that d-offset, and therefore - // // R15, is unchanged. - // } - CMPQ DX, $8 - JGE fixUpSlowForwardCopy - MOVQ (R15), BX - MOVQ BX, (DI) - SUBQ DX, CX - ADDQ DX, DI - ADDQ DX, DX - JMP makeOffsetAtLeast8 - -fixUpSlowForwardCopy: - // !!! Add length (which might be negative now) to d (implied by DI being - // &dst[d]) so that d ends up at the right place when we jump back to the - // top of the loop. Before we do that, though, we save DI to AX so that, if - // length is positive, copying the remaining length bytes will write to the - // right place. - MOVQ DI, AX - ADDQ CX, DI - -finishSlowForwardCopy: - // !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative - // length means that we overrun, but as above, that will be fixed up by - // subsequent iterations of the outermost loop. - CMPQ CX, $0 - JLE loop - MOVQ (R15), BX - MOVQ BX, (AX) - ADDQ $8, R15 - ADDQ $8, AX - SUBQ $8, CX - JMP finishSlowForwardCopy - -verySlowForwardCopy: - // verySlowForwardCopy is a simple implementation of forward copy. In C - // parlance, this is a do/while loop instead of a while loop, since we know - // that length > 0. In Go syntax: - // - // for { - // dst[d] = dst[d - offset] - // d++ - // length-- - // if length == 0 { - // break - // } - // } - MOVB (R15), BX - MOVB BX, (DI) - INCQ R15 - INCQ DI - DECQ CX - JNZ verySlowForwardCopy - JMP loop - -// The code above handles copy tags. -// ---------------------------------------- - -end: - // This is the end of the "for s < len(src)". - // - // if d != len(dst) { etc } - CMPQ DI, R10 - JNE errCorrupt - - // return 0 - MOVQ $0, ret+48(FP) - RET - -errCorrupt: - // return decodeErrCodeCorrupt - MOVQ $1, ret+48(FP) - RET - -errUC4T: - // return decodeErrCodeUnsupportedCopy4Tag - MOVQ $3, ret+48(FP) - RET diff --git a/vendor/github.com/golang/snappy/decode_other.go b/vendor/github.com/golang/snappy/decode_other.go deleted file mode 100644 index 1a8114ab..00000000 --- a/vendor/github.com/golang/snappy/decode_other.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !amd64 - -package snappy - -// decode writes the decoding of src to dst. It assumes that the varint-encoded -// length of the decompressed bytes has already been read, and that len(dst) -// equals that length. -// -// It returns 0 on success or a decodeErrCodeXxx error code on failure. -func decode(dst, src []byte) int { - var d, s, offset, length int - for s < len(src) { - switch src[s] & 0x03 { - case tagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-1]) - case x == 61: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-2]) | uint32(src[s-1])<<8 - case x == 62: - s += 4 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - case x == 63: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - } - length = int(x) + 1 - if length <= 0 { - return decodeErrCodeUnsupportedLiteralLength - } - if length > len(dst)-d || length > len(src)-s { - return decodeErrCodeCorrupt - } - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 4 + int(src[s-2])>>2&0x7 - offset = int(src[s-2])&0xe0<<3 | int(src[s-1]) - - case tagCopy2: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-3])>>2 - offset = int(src[s-2]) | int(src[s-1])<<8 - - case tagCopy4: - return decodeErrCodeUnsupportedCopy4Tag - } - - if offset <= 0 || d < offset || length > len(dst)-d { - return decodeErrCodeCorrupt - } - // Copy from an earlier sub-slice of dst to a later sub-slice. Unlike - // the built-in copy function, this byte-by-byte copy always runs - // forwards, even if the slices overlap. Conceptually, this is: - // - // d += forwardCopy(dst[d:d+length], dst[d-offset:]) - for end := d + length; d != end; d++ { - dst[d] = dst[d-offset] - } - } - if d != len(dst) { - return decodeErrCodeCorrupt - } - return 0 -} diff --git a/vendor/github.com/golang/snappy/encode.go b/vendor/github.com/golang/snappy/encode.go deleted file mode 100644 index 38ebe952..00000000 --- a/vendor/github.com/golang/snappy/encode.go +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -import ( - "encoding/binary" - "errors" - "io" -) - -// maxOffset limits how far copy back-references can go, the same as the C++ -// code. -const maxOffset = 1 << 15 - -// emitLiteral writes a literal chunk and returns the number of bytes written. -func emitLiteral(dst, lit []byte) int { - i, n := 0, uint(len(lit)-1) - switch { - case n < 60: - dst[0] = uint8(n)<<2 | tagLiteral - i = 1 - case n < 1<<8: - dst[0] = 60<<2 | tagLiteral - dst[1] = uint8(n) - i = 2 - case n < 1<<16: - dst[0] = 61<<2 | tagLiteral - dst[1] = uint8(n) - dst[2] = uint8(n >> 8) - i = 3 - case n < 1<<24: - dst[0] = 62<<2 | tagLiteral - dst[1] = uint8(n) - dst[2] = uint8(n >> 8) - dst[3] = uint8(n >> 16) - i = 4 - case int64(n) < 1<<32: - dst[0] = 63<<2 | tagLiteral - dst[1] = uint8(n) - dst[2] = uint8(n >> 8) - dst[3] = uint8(n >> 16) - dst[4] = uint8(n >> 24) - i = 5 - default: - panic("snappy: source buffer is too long") - } - if copy(dst[i:], lit) != len(lit) { - panic("snappy: destination buffer is too short") - } - return i + len(lit) -} - -// emitCopy writes a copy chunk and returns the number of bytes written. -func emitCopy(dst []byte, offset, length int32) int { - i := 0 - for length > 0 { - x := length - 4 - if 0 <= x && x < 1<<3 && offset < 1<<11 { - dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1 - dst[i+1] = uint8(offset) - i += 2 - break - } - - x = length - if x > 1<<6 { - x = 1 << 6 - } - dst[i+0] = uint8(x-1)<<2 | tagCopy2 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - i += 3 - length -= x - } - return i -} - -// Encode returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// It is valid to pass a nil dst. -func Encode(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - for len(src) > 0 { - p := src - src = nil - if len(p) > maxBlockSize { - p, src = p[:maxBlockSize], p[maxBlockSize:] - } - d += encodeBlock(dst[d:], p) - } - return dst[:d] -} - -// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// len(dst) >= MaxEncodedLen(len(src)) && -// 0 < len(src) && len(src) <= maxBlockSize -func encodeBlock(dst, src []byte) (d int) { - // Return early if src is short. - if len(src) <= 4 { - return emitLiteral(dst, src) - } - - // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. - const maxTableSize = 1 << 14 - shift, tableSize := uint(32-8), 1<<8 - for tableSize < maxTableSize && tableSize < len(src) { - shift-- - tableSize *= 2 - } - var table [maxTableSize]int32 - - // Iterate over the source bytes. - var ( - s int32 // The iterator position. - t int32 // The last position with the same hash as s. - lit int32 // The start position of any pending literal bytes. - - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned, look at every third byte, etc.. When a match is found, - // immediately go back to looking at every byte. This is a small loss - // (~5% performance, ~0.1% density) for compressible data due to more - // bookkeeping, but for non-compressible data (such as JPEG) it's a - // huge win since the compressor quickly "realizes" the data is - // incompressible and doesn't bother looking for matches everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip uint32 = 32 - ) - for uint32(s+3) < uint32(len(src)) { // The uint32 conversions catch overflow from the +3. - // Update the hash table. - b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3] - h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24 - p := &table[(h*0x1e35a7bd)>>shift] - // We need to to store values in [-1, inf) in table. To save - // some initialization time, (re)use the table's zero value - // and shift the values against this zero: add 1 on writes, - // subtract 1 on reads. - t, *p = *p-1, s+1 - // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. - if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] { - s += int32(skip >> 5) - skip++ - continue - } - skip = 32 - // Otherwise, we have a match. First, emit any pending literal bytes. - if lit != s { - d += emitLiteral(dst[d:], src[lit:s]) - } - // Extend the match to be as long as possible. - s0 := s - s, t = s+4, t+4 - for int(s) < len(src) && src[s] == src[t] { - s++ - t++ - } - // Emit the copied bytes. - d += emitCopy(dst[d:], s-t, s-s0) - lit = s - } - - // Emit any final pending literal bytes and return. - if int(lit) != len(src) { - d += emitLiteral(dst[d:], src[lit:]) - } - return d -} - -// MaxEncodedLen returns the maximum length of a snappy block, given its -// uncompressed length. -// -// It will return a negative value if srcLen is too large to encode. -func MaxEncodedLen(srcLen int) int { - n := uint64(srcLen) - if n > 0xffffffff { - return -1 - } - // Compressed data can be defined as: - // compressed := item* literal* - // item := literal* copy - // - // The trailing literal sequence has a space blowup of at most 62/60 - // since a literal of length 60 needs one tag byte + one extra byte - // for length information. - // - // Item blowup is trickier to measure. Suppose the "copy" op copies - // 4 bytes of data. Because of a special check in the encoding code, - // we produce a 4-byte copy only if the offset is < 65536. Therefore - // the copy op takes 3 bytes to encode, and this type of item leads - // to at most the 62/60 blowup for representing literals. - // - // Suppose the "copy" op copies 5 bytes of data. If the offset is big - // enough, it will take 5 bytes to encode the copy op. Therefore the - // worst case here is a one-byte literal followed by a five-byte copy. - // That is, 6 bytes of input turn into 7 bytes of "compressed" data. - // - // This last factor dominates the blowup, so the final estimate is: - n = 32 + n + n/6 - if n > 0xffffffff { - return -1 - } - return int(n) -} - -var errClosed = errors.New("snappy: Writer is closed") - -// NewWriter returns a new Writer that compresses to w. -// -// The Writer returned does not buffer writes. There is no need to Flush or -// Close such a Writer. -// -// Deprecated: the Writer returned is not suitable for many small writes, only -// for few large writes. Use NewBufferedWriter instead, which is efficient -// regardless of the frequency and shape of the writes, and remember to Close -// that Writer when done. -func NewWriter(w io.Writer) *Writer { - return &Writer{ - w: w, - obuf: make([]byte, obufLen), - } -} - -// NewBufferedWriter returns a new Writer that compresses to w, using the -// framing format described at -// https://github.com/google/snappy/blob/master/framing_format.txt -// -// The Writer returned buffers writes. Users must call Close to guarantee all -// data has been forwarded to the underlying io.Writer. They may also call -// Flush zero or more times before calling Close. -func NewBufferedWriter(w io.Writer) *Writer { - return &Writer{ - w: w, - ibuf: make([]byte, 0, maxBlockSize), - obuf: make([]byte, obufLen), - } -} - -// Writer is an io.Writer than can write Snappy-compressed bytes. -type Writer struct { - w io.Writer - err error - - // ibuf is a buffer for the incoming (uncompressed) bytes. - // - // Its use is optional. For backwards compatibility, Writers created by the - // NewWriter function have ibuf == nil, do not buffer incoming bytes, and - // therefore do not need to be Flush'ed or Close'd. - ibuf []byte - - // obuf is a buffer for the outgoing (compressed) bytes. - obuf []byte - - // wroteStreamHeader is whether we have written the stream header. - wroteStreamHeader bool -} - -// Reset discards the writer's state and switches the Snappy writer to write to -// w. This permits reusing a Writer rather than allocating a new one. -func (w *Writer) Reset(writer io.Writer) { - w.w = writer - w.err = nil - if w.ibuf != nil { - w.ibuf = w.ibuf[:0] - } - w.wroteStreamHeader = false -} - -// Write satisfies the io.Writer interface. -func (w *Writer) Write(p []byte) (nRet int, errRet error) { - if w.ibuf == nil { - // Do not buffer incoming bytes. This does not perform or compress well - // if the caller of Writer.Write writes many small slices. This - // behavior is therefore deprecated, but still supported for backwards - // compatibility with code that doesn't explicitly Flush or Close. - return w.write(p) - } - - // The remainder of this method is based on bufio.Writer.Write from the - // standard library. - - for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err == nil { - var n int - if len(w.ibuf) == 0 { - // Large write, empty buffer. - // Write directly from p to avoid copy. - n, _ = w.write(p) - } else { - n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - w.Flush() - } - nRet += n - p = p[n:] - } - if w.err != nil { - return nRet, w.err - } - n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - nRet += n - return nRet, nil -} - -func (w *Writer) write(p []byte) (nRet int, errRet error) { - if w.err != nil { - return 0, w.err - } - for len(p) > 0 { - obufStart := len(magicChunk) - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - copy(w.obuf, magicChunk) - obufStart = 0 - } - - var uncompressed []byte - if len(p) > maxBlockSize { - uncompressed, p = p[:maxBlockSize], p[maxBlockSize:] - } else { - uncompressed, p = p, nil - } - checksum := crc(uncompressed) - - // Compress the buffer, discarding the result if the improvement - // isn't at least 12.5%. - compressed := Encode(w.obuf[obufHeaderLen:], uncompressed) - chunkType := uint8(chunkTypeCompressedData) - chunkLen := 4 + len(compressed) - obufEnd := obufHeaderLen + len(compressed) - if len(compressed) >= len(uncompressed)-len(uncompressed)/8 { - chunkType = chunkTypeUncompressedData - chunkLen = 4 + len(uncompressed) - obufEnd = obufHeaderLen - } - - // Fill in the per-chunk header that comes before the body. - w.obuf[len(magicChunk)+0] = chunkType - w.obuf[len(magicChunk)+1] = uint8(chunkLen >> 0) - w.obuf[len(magicChunk)+2] = uint8(chunkLen >> 8) - w.obuf[len(magicChunk)+3] = uint8(chunkLen >> 16) - w.obuf[len(magicChunk)+4] = uint8(checksum >> 0) - w.obuf[len(magicChunk)+5] = uint8(checksum >> 8) - w.obuf[len(magicChunk)+6] = uint8(checksum >> 16) - w.obuf[len(magicChunk)+7] = uint8(checksum >> 24) - - if _, err := w.w.Write(w.obuf[obufStart:obufEnd]); err != nil { - w.err = err - return nRet, err - } - if chunkType == chunkTypeUncompressedData { - if _, err := w.w.Write(uncompressed); err != nil { - w.err = err - return nRet, err - } - } - nRet += len(uncompressed) - } - return nRet, nil -} - -// Flush flushes the Writer to its underlying io.Writer. -func (w *Writer) Flush() error { - if w.err != nil { - return w.err - } - if len(w.ibuf) == 0 { - return nil - } - w.write(w.ibuf) - w.ibuf = w.ibuf[:0] - return w.err -} - -// Close calls Flush and then closes the Writer. -func (w *Writer) Close() error { - w.Flush() - ret := w.err - if w.err == nil { - w.err = errClosed - } - return ret -} diff --git a/vendor/github.com/golang/snappy/snappy.go b/vendor/github.com/golang/snappy/snappy.go deleted file mode 100644 index 01025426..00000000 --- a/vendor/github.com/golang/snappy/snappy.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package snappy implements the snappy block-based compression format. -// It aims for very high speeds and reasonable compression. -// -// The C++ snappy implementation is at https://github.com/google/snappy -package snappy // import "github.com/golang/snappy" - -import ( - "hash/crc32" -) - -/* -Each encoded block begins with the varint-encoded length of the decoded data, -followed by a sequence of chunks. Chunks begin and end on byte boundaries. The -first byte of each chunk is broken into its 2 least and 6 most significant bits -called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. -Zero means a literal tag. All other values mean a copy tag. - -For literal tags: - - If m < 60, the next 1 + m bytes are literal bytes. - - Otherwise, let n be the little-endian unsigned integer denoted by the next - m - 59 bytes. The next 1 + n bytes after that are literal bytes. - -For copy tags, length bytes are copied from offset bytes ago, in the style of -Lempel-Ziv compression algorithms. In particular: - - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). - The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 - of the offset. The next byte is bits 0-7 of the offset. - - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). - The length is 1 + m. The offset is the little-endian unsigned integer - denoted by the next 2 bytes. - - For l == 3, this tag is a legacy format that is no longer supported. -*/ -const ( - tagLiteral = 0x00 - tagCopy1 = 0x01 - tagCopy2 = 0x02 - tagCopy4 = 0x03 -) - -const ( - checksumSize = 4 - chunkHeaderSize = 4 - magicChunk = "\xff\x06\x00\x00" + magicBody - magicBody = "sNaPpY" - - // maxBlockSize is the maximum size of the input to encodeBlock. It is not - // part of the wire format per se, but some parts of the encoder assume - // that an offset fits into a uint16. - // - // Also, for the framing format (Writer type instead of Encode function), - // https://github.com/google/snappy/blob/master/framing_format.txt says - // that "the uncompressed data in a chunk must be no longer than 65536 - // bytes". - maxBlockSize = 65536 - - // maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is - // hard coded to be a const instead of a variable, so that obufLen can also - // be a const. Their equivalence is confirmed by - // TestMaxEncodedLenOfMaxBlockSize. - maxEncodedLenOfMaxBlockSize = 76490 - - obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize - obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize -) - -const ( - chunkTypeCompressedData = 0x00 - chunkTypeUncompressedData = 0x01 - chunkTypePadding = 0xfe - chunkTypeStreamIdentifier = 0xff -) - -var crcTable = crc32.MakeTable(crc32.Castagnoli) - -// crc implements the checksum specified in section 3 of -// https://github.com/google/snappy/blob/master/framing_format.txt -func crc(b []byte) uint32 { - c := crc32.Update(0, crcTable, b) - return uint32(c>>15|c<<17) + 0xa282ead8 -} diff --git a/vendor/github.com/golang/snappy/snappy_test.go b/vendor/github.com/golang/snappy/snappy_test.go deleted file mode 100644 index 65c0013b..00000000 --- a/vendor/github.com/golang/snappy/snappy_test.go +++ /dev/null @@ -1,973 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -import ( - "bytes" - "encoding/binary" - "flag" - "fmt" - "io" - "io/ioutil" - "math/rand" - "net/http" - "os" - "path/filepath" - "strings" - "testing" -) - -var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") - -func TestMaxEncodedLenOfMaxBlockSize(t *testing.T) { - got := maxEncodedLenOfMaxBlockSize - want := MaxEncodedLen(maxBlockSize) - if got != want { - t.Fatalf("got %d, want %d", got, want) - } -} - -func cmp(a, b []byte) error { - if bytes.Equal(a, b) { - return nil - } - if len(a) != len(b) { - return fmt.Errorf("got %d bytes, want %d", len(a), len(b)) - } - for i := range a { - if a[i] != b[i] { - return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i]) - } - } - return nil -} - -func roundtrip(b, ebuf, dbuf []byte) error { - d, err := Decode(dbuf, Encode(ebuf, b)) - if err != nil { - return fmt.Errorf("decoding error: %v", err) - } - if err := cmp(d, b); err != nil { - return fmt.Errorf("roundtrip mismatch: %v", err) - } - return nil -} - -func TestEmpty(t *testing.T) { - if err := roundtrip(nil, nil, nil); err != nil { - t.Fatal(err) - } -} - -func TestSmallCopy(t *testing.T) { - for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} { - for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} { - for i := 0; i < 32; i++ { - s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb" - if err := roundtrip([]byte(s), ebuf, dbuf); err != nil { - t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err) - } - } - } - } -} - -func TestSmallRand(t *testing.T) { - rng := rand.New(rand.NewSource(1)) - for n := 1; n < 20000; n += 23 { - b := make([]byte, n) - for i := range b { - b[i] = uint8(rng.Intn(256)) - } - if err := roundtrip(b, nil, nil); err != nil { - t.Fatal(err) - } - } -} - -func TestSmallRegular(t *testing.T) { - for n := 1; n < 20000; n += 23 { - b := make([]byte, n) - for i := range b { - b[i] = uint8(i%10 + 'a') - } - if err := roundtrip(b, nil, nil); err != nil { - t.Fatal(err) - } - } -} - -func TestInvalidVarint(t *testing.T) { - testCases := []struct { - desc string - input string - }{{ - "invalid varint, final byte has continuation bit set", - "\xff", - }, { - "invalid varint, value overflows uint64", - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00", - }, { - // https://github.com/google/snappy/blob/master/format_description.txt - // says that "the stream starts with the uncompressed length [as a - // varint] (up to a maximum of 2^32 - 1)". - "valid varint (as uint64), but value overflows uint32", - "\x80\x80\x80\x80\x10", - }} - - for _, tc := range testCases { - input := []byte(tc.input) - if _, err := DecodedLen(input); err != ErrCorrupt { - t.Errorf("%s: DecodedLen: got %v, want ErrCorrupt", tc.desc, err) - } - if _, err := Decode(nil, input); err != ErrCorrupt { - t.Errorf("%s: Decode: got %v, want ErrCorrupt", tc.desc, err) - } - } -} - -func TestDecode(t *testing.T) { - lit40Bytes := make([]byte, 40) - for i := range lit40Bytes { - lit40Bytes[i] = byte(i) - } - lit40 := string(lit40Bytes) - - testCases := []struct { - desc string - input string - want string - wantErr error - }{{ - `decodedLen=0; valid input`, - "\x00", - "", - nil, - }, { - `decodedLen=3; tagLiteral, 0-byte length; length=3; valid input`, - "\x03" + "\x08\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=2; tagLiteral, 0-byte length; length=3; not enough dst bytes`, - "\x02" + "\x08\xff\xff\xff", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 0-byte length; length=3; not enough src bytes`, - "\x03" + "\x08\xff\xff", - "", - ErrCorrupt, - }, { - `decodedLen=40; tagLiteral, 0-byte length; length=40; valid input`, - "\x28" + "\x9c" + lit40, - lit40, - nil, - }, { - `decodedLen=1; tagLiteral, 1-byte length; not enough length bytes`, - "\x01" + "\xf0", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 1-byte length; length=3; valid input`, - "\x03" + "\xf0\x02\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=1; tagLiteral, 2-byte length; not enough length bytes`, - "\x01" + "\xf4\x00", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 2-byte length; length=3; valid input`, - "\x03" + "\xf4\x02\x00\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=1; tagLiteral, 3-byte length; not enough length bytes`, - "\x01" + "\xf8\x00\x00", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 3-byte length; length=3; valid input`, - "\x03" + "\xf8\x02\x00\x00\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=1; tagLiteral, 4-byte length; not enough length bytes`, - "\x01" + "\xfc\x00\x00\x00", - "", - ErrCorrupt, - }, { - `decodedLen=1; tagLiteral, 4-byte length; length=3; not enough dst bytes`, - "\x01" + "\xfc\x02\x00\x00\x00\xff\xff\xff", - "", - ErrCorrupt, - }, { - `decodedLen=4; tagLiteral, 4-byte length; length=3; not enough src bytes`, - "\x04" + "\xfc\x02\x00\x00\x00\xff", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 4-byte length; length=3; valid input`, - "\x03" + "\xfc\x02\x00\x00\x00\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=4; tagCopy1, 1 extra length|offset byte; not enough extra bytes`, - "\x04" + "\x01", - "", - ErrCorrupt, - }, { - `decodedLen=4; tagCopy2, 2 extra length|offset bytes; not enough extra bytes`, - "\x04" + "\x02\x00", - "", - ErrCorrupt, - }, { - `decodedLen=4; tagCopy4; unsupported COPY_4 tag`, - "\x04" + "\x03\x00\x00\x00\x00", - "", - errUnsupportedCopy4Tag, - }, { - `decodedLen=4; tagLiteral (4 bytes "abcd"); valid input`, - "\x04" + "\x0cabcd", - "abcd", - nil, - }, { - `decodedLen=13; tagLiteral (4 bytes "abcd"); tagCopy1; length=9 offset=4; valid input`, - "\x0d" + "\x0cabcd" + "\x15\x04", - "abcdabcdabcda", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; valid input`, - "\x08" + "\x0cabcd" + "\x01\x04", - "abcdabcd", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=2; valid input`, - "\x08" + "\x0cabcd" + "\x01\x02", - "abcdcdcd", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=1; valid input`, - "\x08" + "\x0cabcd" + "\x01\x01", - "abcddddd", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=0; zero offset`, - "\x08" + "\x0cabcd" + "\x01\x00", - "", - ErrCorrupt, - }, { - `decodedLen=9; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; inconsistent dLen`, - "\x09" + "\x0cabcd" + "\x01\x04", - "", - ErrCorrupt, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=5; offset too large`, - "\x08" + "\x0cabcd" + "\x01\x05", - "", - ErrCorrupt, - }, { - `decodedLen=7; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; length too large`, - "\x07" + "\x0cabcd" + "\x01\x04", - "", - ErrCorrupt, - }, { - `decodedLen=6; tagLiteral (4 bytes "abcd"); tagCopy2; length=2 offset=3; valid input`, - "\x06" + "\x0cabcd" + "\x06\x03\x00", - "abcdbc", - nil, - }} - - const ( - // notPresentXxx defines a range of byte values [0xa0, 0xc5) that are - // not present in either the input or the output. It is written to dBuf - // to check that Decode does not write bytes past the end of - // dBuf[:dLen]. - // - // The magic number 37 was chosen because it is prime. A more 'natural' - // number like 32 might lead to a false negative if, for example, a - // byte was incorrectly copied 4*8 bytes later. - notPresentBase = 0xa0 - notPresentLen = 37 - ) - - var dBuf [100]byte -loop: - for i, tc := range testCases { - input := []byte(tc.input) - for _, x := range input { - if notPresentBase <= x && x < notPresentBase+notPresentLen { - t.Errorf("#%d (%s): input shouldn't contain %#02x\ninput: % x", i, tc.desc, x, input) - continue loop - } - } - - dLen, n := binary.Uvarint(input) - if n <= 0 { - t.Errorf("#%d (%s): invalid varint-encoded dLen", i, tc.desc) - continue - } - if dLen > uint64(len(dBuf)) { - t.Errorf("#%d (%s): dLen %d is too large", i, tc.desc, dLen) - continue - } - - for j := range dBuf { - dBuf[j] = byte(notPresentBase + j%notPresentLen) - } - g, gotErr := Decode(dBuf[:], input) - if got := string(g); got != tc.want || gotErr != tc.wantErr { - t.Errorf("#%d (%s):\ngot %q, %v\nwant %q, %v", - i, tc.desc, got, gotErr, tc.want, tc.wantErr) - continue - } - for j, x := range dBuf { - if uint64(j) < dLen { - continue - } - if w := byte(notPresentBase + j%notPresentLen); x != w { - t.Errorf("#%d (%s): Decode overrun: dBuf[%d] was modified: got %#02x, want %#02x\ndBuf: % x", - i, tc.desc, j, x, w, dBuf) - continue loop - } - } - } -} - -// TestDecodeLengthOffset tests decoding an encoding of the form literal + -// copy-length-offset + literal. For example: "abcdefghijkl" + "efghij" + "AB". -func TestDecodeLengthOffset(t *testing.T) { - const ( - prefix = "abcdefghijklmnopqr" - suffix = "ABCDEFGHIJKLMNOPQR" - - // notPresentXxx defines a range of byte values [0xa0, 0xc5) that are - // not present in either the input or the output. It is written to - // gotBuf to check that Decode does not write bytes past the end of - // gotBuf[:totalLen]. - // - // The magic number 37 was chosen because it is prime. A more 'natural' - // number like 32 might lead to a false negative if, for example, a - // byte was incorrectly copied 4*8 bytes later. - notPresentBase = 0xa0 - notPresentLen = 37 - ) - var gotBuf, wantBuf, inputBuf [128]byte - for length := 1; length <= 18; length++ { - for offset := 1; offset <= 18; offset++ { - loop: - for suffixLen := 0; suffixLen <= 18; suffixLen++ { - totalLen := len(prefix) + length + suffixLen - - inputLen := binary.PutUvarint(inputBuf[:], uint64(totalLen)) - inputBuf[inputLen] = tagLiteral + 4*byte(len(prefix)-1) - inputLen++ - inputLen += copy(inputBuf[inputLen:], prefix) - inputBuf[inputLen+0] = tagCopy2 + 4*byte(length-1) - inputBuf[inputLen+1] = byte(offset) - inputBuf[inputLen+2] = 0x00 - inputLen += 3 - if suffixLen > 0 { - inputBuf[inputLen] = tagLiteral + 4*byte(suffixLen-1) - inputLen++ - inputLen += copy(inputBuf[inputLen:], suffix[:suffixLen]) - } - input := inputBuf[:inputLen] - - for i := range gotBuf { - gotBuf[i] = byte(notPresentBase + i%notPresentLen) - } - got, err := Decode(gotBuf[:], input) - if err != nil { - t.Errorf("length=%d, offset=%d; suffixLen=%d: %v", length, offset, suffixLen, err) - continue - } - - wantLen := 0 - wantLen += copy(wantBuf[wantLen:], prefix) - for i := 0; i < length; i++ { - wantBuf[wantLen] = wantBuf[wantLen-offset] - wantLen++ - } - wantLen += copy(wantBuf[wantLen:], suffix[:suffixLen]) - want := wantBuf[:wantLen] - - for _, x := range input { - if notPresentBase <= x && x < notPresentBase+notPresentLen { - t.Errorf("length=%d, offset=%d; suffixLen=%d: input shouldn't contain %#02x\ninput: % x", - length, offset, suffixLen, x, input) - continue loop - } - } - for i, x := range gotBuf { - if i < totalLen { - continue - } - if w := byte(notPresentBase + i%notPresentLen); x != w { - t.Errorf("length=%d, offset=%d; suffixLen=%d; totalLen=%d: "+ - "Decode overrun: gotBuf[%d] was modified: got %#02x, want %#02x\ngotBuf: % x", - length, offset, suffixLen, totalLen, i, x, w, gotBuf) - continue loop - } - } - for _, x := range want { - if notPresentBase <= x && x < notPresentBase+notPresentLen { - t.Errorf("length=%d, offset=%d; suffixLen=%d: want shouldn't contain %#02x\nwant: % x", - length, offset, suffixLen, x, want) - continue loop - } - } - - if !bytes.Equal(got, want) { - t.Errorf("length=%d, offset=%d; suffixLen=%d:\ninput % x\ngot % x\nwant % x", - length, offset, suffixLen, input, got, want) - continue - } - } - } - } -} - -func TestDecodeGoldenInput(t *testing.T) { - src, err := ioutil.ReadFile("testdata/pi.txt.rawsnappy") - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - got, err := Decode(nil, src) - if err != nil { - t.Fatalf("Decode: %v", err) - } - want, err := ioutil.ReadFile("testdata/pi.txt") - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - if err := cmp(got, want); err != nil { - t.Fatal(err) - } -} - -// TestSlowForwardCopyOverrun tests the "expand the pattern" algorithm -// described in decode_amd64.s and its claim of a 10 byte overrun worst case. -func TestSlowForwardCopyOverrun(t *testing.T) { - const base = 100 - - for length := 1; length < 18; length++ { - for offset := 1; offset < 18; offset++ { - highWaterMark := base - d := base - l := length - o := offset - - // makeOffsetAtLeast8 - for o < 8 { - if end := d + 8; highWaterMark < end { - highWaterMark = end - } - l -= o - d += o - o += o - } - - // fixUpSlowForwardCopy - a := d - d += l - - // finishSlowForwardCopy - for l > 0 { - if end := a + 8; highWaterMark < end { - highWaterMark = end - } - a += 8 - l -= 8 - } - - dWant := base + length - overrun := highWaterMark - dWant - if d != dWant || overrun < 0 || 10 < overrun { - t.Errorf("length=%d, offset=%d: d and overrun: got (%d, %d), want (%d, something in [0, 10])", - length, offset, d, overrun, dWant) - } - } - } -} - -// TestEncodeNoiseThenRepeats encodes input for which the first half is very -// incompressible and the second half is very compressible. The encoded form's -// length should be closer to 50% of the original length than 100%. -func TestEncodeNoiseThenRepeats(t *testing.T) { - for _, origLen := range []int{32 * 1024, 256 * 1024, 2048 * 1024} { - src := make([]byte, origLen) - rng := rand.New(rand.NewSource(1)) - firstHalf, secondHalf := src[:origLen/2], src[origLen/2:] - for i := range firstHalf { - firstHalf[i] = uint8(rng.Intn(256)) - } - for i := range secondHalf { - secondHalf[i] = uint8(i >> 8) - } - dst := Encode(nil, src) - if got, want := len(dst), origLen*3/4; got >= want { - t.Errorf("origLen=%d: got %d encoded bytes, want less than %d", origLen, got, want) - } - } -} - -func TestFramingFormat(t *testing.T) { - // src is comprised of alternating 1e5-sized sequences of random - // (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen - // because it is larger than maxBlockSize (64k). - src := make([]byte, 1e6) - rng := rand.New(rand.NewSource(1)) - for i := 0; i < 10; i++ { - if i%2 == 0 { - for j := 0; j < 1e5; j++ { - src[1e5*i+j] = uint8(rng.Intn(256)) - } - } else { - for j := 0; j < 1e5; j++ { - src[1e5*i+j] = uint8(i) - } - } - } - - buf := new(bytes.Buffer) - if _, err := NewWriter(buf).Write(src); err != nil { - t.Fatalf("Write: encoding: %v", err) - } - dst, err := ioutil.ReadAll(NewReader(buf)) - if err != nil { - t.Fatalf("ReadAll: decoding: %v", err) - } - if err := cmp(dst, src); err != nil { - t.Fatal(err) - } -} - -func TestWriterGoldenOutput(t *testing.T) { - buf := new(bytes.Buffer) - w := NewBufferedWriter(buf) - defer w.Close() - w.Write([]byte("abcd")) // Not compressible. - w.Flush() - w.Write(bytes.Repeat([]byte{'A'}, 100)) // Compressible. - w.Flush() - got := buf.String() - want := strings.Join([]string{ - magicChunk, - "\x01\x08\x00\x00", // Uncompressed chunk, 8 bytes long (including 4 byte checksum). - "\x68\x10\xe6\xb6", // Checksum. - "\x61\x62\x63\x64", // Uncompressed payload: "abcd". - "\x00\x0d\x00\x00", // Compressed chunk, 13 bytes long (including 4 byte checksum). - "\x37\xcb\xbc\x9d", // Checksum. - "\x64", // Compressed payload: Uncompressed length (varint encoded): 100. - "\x00\x41", // Compressed payload: tagLiteral, length=1, "A". - "\xfe\x01\x00", // Compressed payload: tagCopy2, length=64, offset=1. - "\x8a\x01\x00", // Compressed payload: tagCopy2, length=35, offset=1. - }, "") - if got != want { - t.Fatalf("\ngot: % x\nwant: % x", got, want) - } -} - -func TestNewBufferedWriter(t *testing.T) { - // Test all 32 possible sub-sequences of these 5 input slices. - // - // Their lengths sum to 400,000, which is over 6 times the Writer ibuf - // capacity: 6 * maxBlockSize is 393,216. - inputs := [][]byte{ - bytes.Repeat([]byte{'a'}, 40000), - bytes.Repeat([]byte{'b'}, 150000), - bytes.Repeat([]byte{'c'}, 60000), - bytes.Repeat([]byte{'d'}, 120000), - bytes.Repeat([]byte{'e'}, 30000), - } -loop: - for i := 0; i < 1< 0; { - i := copy(x, src) - x = x[i:] - } - return dst -} - -func benchWords(b *testing.B, n int, decode bool) { - // Note: the file is OS-language dependent so the resulting values are not - // directly comparable for non-US-English OS installations. - data := expand(readFile(b, "/usr/share/dict/words"), n) - if decode { - benchDecode(b, data) - } else { - benchEncode(b, data) - } -} - -func BenchmarkWordsDecode1e1(b *testing.B) { benchWords(b, 1e1, true) } -func BenchmarkWordsDecode1e2(b *testing.B) { benchWords(b, 1e2, true) } -func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) } -func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) } -func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) } -func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) } -func BenchmarkWordsEncode1e1(b *testing.B) { benchWords(b, 1e1, false) } -func BenchmarkWordsEncode1e2(b *testing.B) { benchWords(b, 1e2, false) } -func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) } -func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) } -func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) } -func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) } - -func BenchmarkRandomEncode(b *testing.B) { - rng := rand.New(rand.NewSource(1)) - data := make([]byte, 1<<20) - for i := range data { - data[i] = uint8(rng.Intn(256)) - } - benchEncode(b, data) -} - -// testFiles' values are copied directly from -// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc -// The label field is unused in snappy-go. -var testFiles = []struct { - label string - filename string - sizeLimit int -}{ - {"html", "html", 0}, - {"urls", "urls.10K", 0}, - {"jpg", "fireworks.jpeg", 0}, - {"jpg_200", "fireworks.jpeg", 200}, - {"pdf", "paper-100k.pdf", 0}, - {"html4", "html_x_4", 0}, - {"txt1", "alice29.txt", 0}, - {"txt2", "asyoulik.txt", 0}, - {"txt3", "lcet10.txt", 0}, - {"txt4", "plrabn12.txt", 0}, - {"pb", "geo.protodata", 0}, - {"gaviota", "kppkn.gtb", 0}, -} - -const ( - // The benchmark data files are at this canonical URL. - benchURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/" - - // They are copied to this local directory. - benchDir = "testdata/bench" -) - -func downloadBenchmarkFiles(b *testing.B, basename string) (errRet error) { - filename := filepath.Join(benchDir, basename) - if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 { - return nil - } - - if !*download { - b.Skipf("test data not found; skipping benchmark without the -download flag") - } - // Download the official snappy C++ implementation reference test data - // files for benchmarking. - if err := os.MkdirAll(benchDir, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create %s: %s", benchDir, err) - } - - f, err := os.Create(filename) - if err != nil { - return fmt.Errorf("failed to create %s: %s", filename, err) - } - defer f.Close() - defer func() { - if errRet != nil { - os.Remove(filename) - } - }() - url := benchURL + basename - resp, err := http.Get(url) - if err != nil { - return fmt.Errorf("failed to download %s: %s", url, err) - } - defer resp.Body.Close() - if s := resp.StatusCode; s != http.StatusOK { - return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s)) - } - _, err = io.Copy(f, resp.Body) - if err != nil { - return fmt.Errorf("failed to download %s to %s: %s", url, filename, err) - } - return nil -} - -func benchFile(b *testing.B, n int, decode bool) { - if err := downloadBenchmarkFiles(b, testFiles[n].filename); err != nil { - b.Fatalf("failed to download testdata: %s", err) - } - data := readFile(b, filepath.Join(benchDir, testFiles[n].filename)) - if n := testFiles[n].sizeLimit; 0 < n && n < len(data) { - data = data[:n] - } - if decode { - benchDecode(b, data) - } else { - benchEncode(b, data) - } -} - -// Naming convention is kept similar to what snappy's C++ implementation uses. -func Benchmark_UFlat0(b *testing.B) { benchFile(b, 0, true) } -func Benchmark_UFlat1(b *testing.B) { benchFile(b, 1, true) } -func Benchmark_UFlat2(b *testing.B) { benchFile(b, 2, true) } -func Benchmark_UFlat3(b *testing.B) { benchFile(b, 3, true) } -func Benchmark_UFlat4(b *testing.B) { benchFile(b, 4, true) } -func Benchmark_UFlat5(b *testing.B) { benchFile(b, 5, true) } -func Benchmark_UFlat6(b *testing.B) { benchFile(b, 6, true) } -func Benchmark_UFlat7(b *testing.B) { benchFile(b, 7, true) } -func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) } -func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) } -func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) } -func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) } -func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) } -func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) } -func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) } -func Benchmark_ZFlat3(b *testing.B) { benchFile(b, 3, false) } -func Benchmark_ZFlat4(b *testing.B) { benchFile(b, 4, false) } -func Benchmark_ZFlat5(b *testing.B) { benchFile(b, 5, false) } -func Benchmark_ZFlat6(b *testing.B) { benchFile(b, 6, false) } -func Benchmark_ZFlat7(b *testing.B) { benchFile(b, 7, false) } -func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) } -func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) } -func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) } -func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) } diff --git a/vendor/github.com/golang/snappy/testdata/pi.txt b/vendor/github.com/golang/snappy/testdata/pi.txt deleted file mode 100644 index 563af418..00000000 --- a/vendor/github.com/golang/snappy/testdata/pi.txt +++ /dev/null @@ -1 +0,0 @@ -3.14159265358979323846264338327950288419716939937510582097494459230781640628620899862803482534211706798214808651328230664709384460955058223172535940812848111745028410270193852110555964462294895493038196442881097566593344612847564823378678316527120190914564856692346034861045432664821339360726024914127372458700660631558817488152092096282925409171536436789259036001133053054882046652138414695194151160943305727036575959195309218611738193261179310511854807446237996274956735188575272489122793818301194912983367336244065664308602139494639522473719070217986094370277053921717629317675238467481846766940513200056812714526356082778577134275778960917363717872146844090122495343014654958537105079227968925892354201995611212902196086403441815981362977477130996051870721134999999837297804995105973173281609631859502445945534690830264252230825334468503526193118817101000313783875288658753320838142061717766914730359825349042875546873115956286388235378759375195778185778053217122680661300192787661119590921642019893809525720106548586327886593615338182796823030195203530185296899577362259941389124972177528347913151557485724245415069595082953311686172785588907509838175463746493931925506040092770167113900984882401285836160356370766010471018194295559619894676783744944825537977472684710404753464620804668425906949129331367702898915210475216205696602405803815019351125338243003558764024749647326391419927260426992279678235478163600934172164121992458631503028618297455570674983850549458858692699569092721079750930295532116534498720275596023648066549911988183479775356636980742654252786255181841757467289097777279380008164706001614524919217321721477235014144197356854816136115735255213347574184946843852332390739414333454776241686251898356948556209921922218427255025425688767179049460165346680498862723279178608578438382796797668145410095388378636095068006422512520511739298489608412848862694560424196528502221066118630674427862203919494504712371378696095636437191728746776465757396241389086583264599581339047802759009946576407895126946839835259570982582262052248940772671947826848260147699090264013639443745530506820349625245174939965143142980919065925093722169646151570985838741059788595977297549893016175392846813826868386894277415599185592524595395943104997252468084598727364469584865383673622262609912460805124388439045124413654976278079771569143599770012961608944169486855584840635342207222582848864815845602850601684273945226746767889525213852254995466672782398645659611635488623057745649803559363456817432411251507606947945109659609402522887971089314566913686722874894056010150330861792868092087476091782493858900971490967598526136554978189312978482168299894872265880485756401427047755513237964145152374623436454285844479526586782105114135473573952311342716610213596953623144295248493718711014576540359027993440374200731057853906219838744780847848968332144571386875194350643021845319104848100537061468067491927819119793995206141966342875444064374512371819217999839101591956181467514269123974894090718649423196156794520809514655022523160388193014209376213785595663893778708303906979207734672218256259966150142150306803844773454920260541466592520149744285073251866600213243408819071048633173464965145390579626856100550810665879699816357473638405257145910289706414011097120628043903975951567715770042033786993600723055876317635942187312514712053292819182618612586732157919841484882916447060957527069572209175671167229109816909152801735067127485832228718352093539657251210835791513698820914442100675103346711031412671113699086585163983150197016515116851714376576183515565088490998985998238734552833163550764791853589322618548963213293308985706420467525907091548141654985946163718027098199430992448895757128289059232332609729971208443357326548938239119325974636673058360414281388303203824903758985243744170291327656180937734440307074692112019130203303801976211011004492932151608424448596376698389522868478312355265821314495768572624334418930396864262434107732269780280731891544110104468232527162010526522721116603966655730925471105578537634668206531098965269186205647693125705863566201855810072936065987648611791045334885034611365768675324944166803962657978771855608455296541266540853061434443185867697514566140680070023787765913440171274947042056223053899456131407112700040785473326993908145466464588079727082668306343285878569830523580893306575740679545716377525420211495576158140025012622859413021647155097925923099079654737612551765675135751782966645477917450112996148903046399471329621073404375189573596145890193897131117904297828564750320319869151402870808599048010941214722131794764777262241425485454033215718530614228813758504306332175182979866223717215916077166925474873898665494945011465406284336639379003976926567214638530673609657120918076383271664162748888007869256029022847210403172118608204190004229661711963779213375751149595015660496318629472654736425230817703675159067350235072835405670403867435136222247715891504953098444893330963408780769325993978054193414473774418426312986080998886874132604721569516239658645730216315981931951673538129741677294786724229246543668009806769282382806899640048243540370141631496589794092432378969070697794223625082216889573837986230015937764716512289357860158816175578297352334460428151262720373431465319777741603199066554187639792933441952154134189948544473456738316249934191318148092777710386387734317720754565453220777092120190516609628049092636019759882816133231666365286193266863360627356763035447762803504507772355471058595487027908143562401451718062464362679456127531813407833033625423278394497538243720583531147711992606381334677687969597030983391307710987040859133746414428227726346594704745878477872019277152807317679077071572134447306057007334924369311383504931631284042512192565179806941135280131470130478164378851852909285452011658393419656213491434159562586586557055269049652098580338507224264829397285847831630577775606888764462482468579260395352773480304802900587607582510474709164396136267604492562742042083208566119062545433721315359584506877246029016187667952406163425225771954291629919306455377991403734043287526288896399587947572917464263574552540790914513571113694109119393251910760208252026187985318877058429725916778131496990090192116971737278476847268608490033770242429165130050051683233643503895170298939223345172201381280696501178440874519601212285993716231301711444846409038906449544400619869075485160263275052983491874078668088183385102283345085048608250393021332197155184306354550076682829493041377655279397517546139539846833936383047461199665385815384205685338621867252334028308711232827892125077126294632295639898989358211674562701021835646220134967151881909730381198004973407239610368540664319395097901906996395524530054505806855019567302292191393391856803449039820595510022635353619204199474553859381023439554495977837790237421617271117236434354394782218185286240851400666044332588856986705431547069657474585503323233421073015459405165537906866273337995851156257843229882737231989875714159578111963583300594087306812160287649628674460477464915995054973742562690104903778198683593814657412680492564879855614537234786733039046883834363465537949864192705638729317487233208376011230299113679386270894387993620162951541337142489283072201269014754668476535761647737946752004907571555278196536213239264061601363581559074220202031872776052772190055614842555187925303435139844253223415762336106425063904975008656271095359194658975141310348227693062474353632569160781547818115284366795706110861533150445212747392454494542368288606134084148637767009612071512491404302725386076482363414334623518975766452164137679690314950191085759844239198629164219399490723623464684411739403265918404437805133389452574239950829659122850855582157250310712570126683024029295252201187267675622041542051618416348475651699981161410100299607838690929160302884002691041407928862150784245167090870006992821206604183718065355672525325675328612910424877618258297651579598470356222629348600341587229805349896502262917487882027342092222453398562647669149055628425039127577102840279980663658254889264880254566101729670266407655904290994568150652653053718294127033693137851786090407086671149655834343476933857817113864558736781230145876871266034891390956200993936103102916161528813843790990423174733639480457593149314052976347574811935670911013775172100803155902485309066920376719220332290943346768514221447737939375170344366199104033751117354719185504644902636551281622882446257591633303910722538374218214088350865739177150968288747826569959957449066175834413752239709683408005355984917541738188399944697486762655165827658483588453142775687900290951702835297163445621296404352311760066510124120065975585127617858382920419748442360800719304576189323492292796501987518721272675079812554709589045563579212210333466974992356302549478024901141952123828153091140790738602515227429958180724716259166854513331239480494707911915326734302824418604142636395480004480026704962482017928964766975831832713142517029692348896276684403232609275249603579964692565049368183609003238092934595889706953653494060340216654437558900456328822505452556405644824651518754711962184439658253375438856909411303150952617937800297412076651479394259029896959469955657612186561967337862362561252163208628692221032748892186543648022967807057656151446320469279068212073883778142335628236089632080682224680122482611771858963814091839036736722208883215137556003727983940041529700287830766709444745601345564172543709069793961225714298946715435784687886144458123145935719849225284716050492212424701412147805734551050080190869960330276347870810817545011930714122339086639383395294257869050764310063835198343893415961318543475464955697810382930971646514384070070736041123735998434522516105070270562352660127648483084076118301305279320542746286540360367453286510570658748822569815793678976697422057505968344086973502014102067235850200724522563265134105592401902742162484391403599895353945909440704691209140938700126456001623742880210927645793106579229552498872758461012648369998922569596881592056001016552563756785 \ No newline at end of file diff --git a/vendor/github.com/golang/snappy/testdata/pi.txt.rawsnappy b/vendor/github.com/golang/snappy/testdata/pi.txt.rawsnappy deleted file mode 100644 index 3378c72434769050f6bc59d5296c237903a0a389..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8929 zcmW-n378#qmBp)|0`=ajzOTBvy8CtVl61w+5TJ$H7X(QGVHtw51p#wIhyueVl(H*}Y#)oD55W*{WS^kG#0O>|A6tf9Hs@F5Bk4}(Rn`69d+s^s-r{H9 z*l>EVl^1$Z=EqSGrCE{%evpQ-ABRDZ27Z!7uAio%mnB}D1z8p(k>^IK?`BDug&`LP zZjyR&=*E5;`)-lx>fZbM zftPSal!b2U`DvJXTpFrBx-j%yKXE-i9C0@{ilQv$GT+a_G>gJ4aD&uShvpKdM^sT9 zM_CZ?33b5LFKMtbU zO}Uz=5)Vuyd`+|1+2Zyf<~d24fQ}d}xS315oFX75enfSnG~y#}h%v&yk2T;~Aik!h6&tOFa&lB}rI6(j_~@MfG?R5oBNfN-1fUq%MEWU+iWH z2LhbVIVR;zqhh0FHN3-!VB*XrfAoY;Zpaom&6hluMsC3AC<>(j44&(8?!W_WK}u#) z;Lg{8bnpX@h+`+8tdr!CU#`v6-FFQ$H}v^bL;{F_$fIJv$pDlD9Grj1ic^)$`I_Q6 zyZvWM{_FJnew>0s4n`7uKE?M`7pzg8ICN$&PN_MK<%1@oB9Up7#58u#Z&I# z8F87*2{}jT9BtDCp`Wds+3~UzxF*HBZW7j1L{-RGmN(~+osepTZkRaF%z-LgiX8L9 z;wBx07UOdXlO6!lDL0{mJ$0*bGUtQZEG{h_hAWR(qVh7GHp`nF4L}kK^#o(IM+gUj zmN+Q4*R#wo=bND)XW6_zDT%X239GM{Gt4Xqy~Mdy^L%wnC3LQyOQJv#*&$O2z>2Cp z_<7uopap=II)73!b#E<-owsZ<4&0y|)$B!-@VZD>3)#-ZU!ey!YhE`Y7c8;%A_G2L2isbzk# zqxzRQbPgD#4ZX0plg@3++e1D=QM?F};ilD6B~%w{URs`{rD65@LZD6&6Ntov$^x*F zz!YJ=<1s!FKs=-u+(aarqyYjSYJ(5*?G;~rU?NUQFMN=>>VAqld(%m3Mt9Oja9Qu= zE=)m_M~Qlj63qL&lVqSG_40>0;3lGFdf72>EsDiuF3rkAY1o=62?^c{%T>hLTc#d3 zmpC9L;0tfisgbK*UR*T}&mv90Y4!q}hJb#ZNAd>^9|lsFQk$hp+{-#JBlu386%&A= zCTi~PY-9wawfyEdpnx{ut=^3W08E{=#pYbPRTtpEdXcAIpvP(8xHb(IC2qM?PkldM zt@yp}GCqRl@Bl8XQOq9{AOTr!R<0j+$v(*qgZ$JQ7n`^Q2@*ME7e8`cwz@(qmBtQhAmmhD$fJZ&ZK(_*UP4D{f zIaN;OUcCeP!~1EziSw`g#Qd@jd4jBC=V=2SPOEb?-^<@n{NhX#B8uax3-uvwzyLo% zcLg%36RB9blE|&PMn`?816P9LSWeRQzz@rlOjzIv z4dnmyaVS231`|JDJWlM;3hY%brD)aFA?YPqc!m^piu+en|4>eBC~3ZU4uVOk(bX*N zZ5YP}(B9Z9^;`X&yPyGX^|R_n6R-f?ozfk0w!SUMqT~r^Lr0KbxlIRcG&9UrebL3? zKowC~t+m4t29MP3stLs_jsOG*X%ZAumZ9UdFe2DUN$mwl3Kfp|aoAIy!b!l}2?~k> zh;%c zWu2)bk!Na|Cr|3zqO|0Xgir)ZoXoy?E}TSb!?7HPV%G%Qd(fsAY2Wg2r`thhQ1DE) zZGe!Kz@S{DM6w5kq^kqmN+N&K)qm0}lr2eu>IEC-@}s!_cP);4YqZeMN6HW7#+EvJ zoRY!O7!J9sdFVP50S>eBSrd}s!pjZ7=Q*Tsc`=ERsl9 z19l999=T~X(M5TvNS5aRB&jmioJ>I#@=h@_uYaDFxOt?xp1j`b;BadPUOZxRdivBE zs++3#txk!o_gYFOe{5-PQtVLsS-cLvX=LA%qG3RpB-X@cWWICfLinflp0pT_(TDdT z6?DXT%?5|)#-r<%%sQYYFX=jnEM}3k>-1tDoocEN%3{NYTU{^7eIoE>xS=yr6vr62 z(a5>f=A_VPZ@xj_1nS{hRKMFU-6LO8CQZ>+h2bCm(+0y)i4m#Uks-WWxWtG_w7 zaEvV*N@VS5Os0?@vx~DJ+t&X`uaeYwee81dY9--Z=O+*WP zM^~cWUu{munaeJHFQ@2ADEAo(p-tj}F$ho#+E+bnlPLHoe@1Q{u<@8UBssdJC zZG-^ZBHQVmdMaY-sLnT_*32mlylqgtB**G%A2)%r>s9ZU zn|Dbej=|5IkE=Iot-~YH*5oNJXX;qKs>?Zd79E*ic@6wn9&4aTv3%InQEh1(m2^-+ zh%B5kq=jB_hym?n{p(D|j=+2#;ajfoQGVMf( zY$RF~=45;2-8ztyzv#3E3XvZx%XEWgP5Un>asT&r0NxT;<~4Cb>OwiT1y?nTLH=!y z4#~^jQmz7_%xv-~>JfX@^OS0q4rpZ?J}gl~PJ{tLD~+3|yL06-=@<~d=f_pifuLNx zj&r2M118AEEj?^vo4l}`XOdCZ=f(&UomHI)5M*alNs3zuNA?;p(6mhEHk|#(VHHLk zER;NUo{|guzPBB(5Qj&6# z9?Fjlp}zoMf?q;D}?=LR$mCoOfe z{!)3sL=GZt-l#WfDaX`vXeYH}Y4VRkQB^9r0i>4{8gYT=o*$c)6i-Q|Ds1eszmBL-dpW(eiiG zX3}8vRmbHghKcIBUhgy|tS+2`!$b>U=gh4xF1u!v&`BcSszHh)SUU5TR$=#s@ z=O>yMT0ayU6~RDSkF|k3M=cMM8MJmJ54KGY=Mp&g>FA>bgA+J)s2}QyWTb&tT2W&z z464iMCf}ENrVcew@0B2%=485wUz5)wxY9V61j%?H<7fW?H)T(wj5x>Y&uhkhwr6rmd*X=^l zEzUW`#+$MKC~np$<4<%3CBOQ|w$L~B59bJ#)I*T-Irlq%0F-+Dv#~Z*87C9U1yFky zSI>hBu!i-5ocJk^+Ym6itiAT>Y4lrzkH(FCImBP_>^WY+-_E<^H2izI0B(m*vJ zEkNkWSw;uf4+RFYxl!J2?IeVHt8T!#Zn+LN>i@F%p7qd>oMz(6a5B|i^5%eyHs#}H z?Li+K(unIX4eo!usnbGoXHzbg4qAfTrqxF^MtMMj&bRb13gp_BmLxJ*NBYGOCi&ZF z+gx3;w!B(LIZxNpR6ZNz+f%;r`Btwnf#Bl88QFsJbpt2_&IuR{ zxpGi`IG`ixz4bbdzOO+=Os8kcS4^YcNy!%jCe{bEZ__YP7dO953xeLOoT@i9e@scM z6>?Zh&TMyBRblA;Uam!1b#jMUG)?G%PScsj;dg4RzM;z?2CYED3$R4hUEO+Pfu%`g z`<51jc}By%Z-Jzs7?$02yWZH*4KL^&1`f)baPX^ejvY+MgD|L|oueIk9+WasC{@?x83egoZ>a|hwt)0LO!|avXALQ-^rvVK_;SwM{w%t9p;|=%kF>9Y~$#&3e0Ia!sh4x-65U zbPyvqwat2?n8+DTC4BT!%|qLo%uBAdf1~`BJY*ONi#z$FEzh+~%sKY(3=MR}dX1L_ zK(qXwk~(+FB`wZ-*|6-;=+dB!!%QZj%%?@-@)_840|^a~cz?2h+R_&NNwm_O~86J|(SZ&LzK`rrQN=MdrYI3^SuGd#^ zYL@S!B)!A*+Tor*G|p0wiJBx^ujth@Eza^qa{xc=#9S0c@`f$%nld~h=e9JYW5p1j zFDsYmObYVOn4H~WfyTmvjXK_s4~z|Bl-N+N!!DPHjdMhH^_p9*Z^+UClNkkOKcgx32+3?2^;$iTXbE;1GBPN$jCy^rL%mwzg#&ss z&Cbs|^vWr>DPjI{@s4cK#rAr#sHp{s95%{4n=e#c$3gfEIgdjJ)dMC2s~@j?#oEQr zKkecl7^{6(-G~e`@qXr3n z#`F6T50+7RI6EFb-}nI#C;xcc8LjITv3u zqC`{vCEkdZ#=V(fma&%C7RBQ>s|q+qwwY#KG@`-4yR_K{^v~!YBvr|FrSn6Il}Ik(EbV>(J*JSW=@C{gcm zn^CYHK9)|eXHj|FBCck~(x%BL26MHn8t(UHCqtghzr-ElMP!&W^yWodC2 zKyR3$4%_uUHVqxuWM6_CTGEgYbvekEg=1O}OQg$(rksQMaYc0bwLK2^v!PQlgn&0zjO`O z$)3E^LIr1ZU!4)Y_M^Q`(UTJbz(ea7JDd(P;xAHZp6i8@)2Z>fAd3~CNS7eCer zZtnDM3fftn#xNvnQ1-Bw=E57OdN+$$v9FcQ$K>25cbnDDRdu}_$2K5KihnoQdiruy zr%dIkVVxCG@9l9kg~ct~C3y(+MZMpcY$LqHS}>;g>korDs?8{0pQ9&|H|2{Fxq!(+ zZb7OtSv$R4AWzb^N%;(KCKB0iW=-i#*Y&({iix~OvP)A=*16)f><^a&GntfWkXvYG?h3%oehO7XDgy^LKIbdk}Vt-k_{GSRgRdh z-iT3vis8qv56T&aEMq>Cz9Swxpd_y5=tvbU zERIsQkVV2&Cr3}!v3}M^!z6B>=`fjF?5Q*Pi!vmS4m#Vi72@?*e!AlKH^|y929;HE zXdP+DHN&;hb7}$nVvx5Q@=c?hv=JWE*jlB_9|!AoQ14h$c*q#*Y8udWtq(AijKi_q zHKNxpa|{g-Ei>ic z2DLb~{@}>`(PFX2n}wNj;E;g<45ay$@`?`QF?+mJ&bEpJ41B#Sdm9|+J)vb;M-6l^ zIZQ5r`hf|TvW0rOA-n1MH8MGqvj23J2ERVqkxK^@-Ny{as?B=isU=@OZ zRU*4~c)^v(e~%ixlZsd+FqH7lS*_DsF9K!8Ap0?COy#!UH)Uv6xtf1jF{)(wWdG0Q z$8rKr%uS2W9V>@3l9?08Ig$LU%SI8| LW{Jieo<{!Send pings and receive pongsYesNo Get the type of a received data messageYesYes, see note 2 Other Features -Limit size of received messageYesNo +Compression ExtensionsExperimentalNo Read message using io.ReaderYesNo, see note 3 Write message using io.WriteCloserYesNo, see note 3 diff --git a/vendor/github.com/gorilla/websocket/bench_test.go b/vendor/github.com/gorilla/websocket/bench_test.go deleted file mode 100644 index f66fc36b..00000000 --- a/vendor/github.com/gorilla/websocket/bench_test.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2014 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket - -import ( - "testing" -) - -func BenchmarkMaskBytes(b *testing.B) { - var key [4]byte - data := make([]byte, 1024) - pos := 0 - for i := 0; i < b.N; i++ { - pos = maskBytes(key, pos, data) - } - b.SetBytes(int64(len(data))) -} diff --git a/vendor/github.com/gorilla/websocket/client.go b/vendor/github.com/gorilla/websocket/client.go index a353e185..43a87c75 100644 --- a/vendor/github.com/gorilla/websocket/client.go +++ b/vendor/github.com/gorilla/websocket/client.go @@ -23,6 +23,8 @@ import ( // invalid. var ErrBadHandshake = errors.New("websocket: bad handshake") +var errInvalidCompression = errors.New("websocket: invalid compression negotiation") + // NewClient creates a new client connection using the given net connection. // The URL u specifies the host and request URI. Use requestHeader to specify // the origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies @@ -64,12 +66,24 @@ type Dialer struct { // HandshakeTimeout specifies the duration for the handshake to complete. HandshakeTimeout time.Duration - // Input and output buffer sizes. If the buffer size is zero, then a - // default value of 4096 is used. + // ReadBufferSize and WriteBufferSize specify I/O buffer sizes. If a buffer + // size is zero, then a useful default size is used. The I/O buffer sizes + // do not limit the size of the messages that can be sent or received. ReadBufferSize, WriteBufferSize int // Subprotocols specifies the client's requested subprotocols. Subprotocols []string + + // EnableCompression specifies if the client should attempt to negotiate + // per message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool + + // Jar specifies the cookie jar. + // If Jar is nil, cookies are not sent in requests and ignored + // in responses. + Jar http.CookieJar } var errMalformedURL = errors.New("malformed ws or wss URL") @@ -83,7 +97,6 @@ func parseURL(s string) (*url.URL, error) { // // ws-URI = "ws:" "//" host [ ":" port ] path [ "?" query ] // wss-URI = "wss:" "//" host [ ":" port ] path [ "?" query ] - var u url.URL switch { case strings.HasPrefix(s, "ws://"): @@ -193,6 +206,13 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re Host: u.Host, } + // Set the cookies present in the cookie jar of the dialer + if d.Jar != nil { + for _, cookie := range d.Jar.Cookies(u) { + req.AddCookie(cookie) + } + } + // Set the request headers using the capitalization for names and values in // RFC examples. Although the capitalization shouldn't matter, there are // servers that depend on it. The Header.Set method is not used because the @@ -214,6 +234,7 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re k == "Connection" || k == "Sec-Websocket-Key" || k == "Sec-Websocket-Version" || + k == "Sec-Websocket-Extensions" || (k == "Sec-Websocket-Protocol" && len(d.Subprotocols) > 0): return nil, nil, errors.New("websocket: duplicate header not allowed: " + k) default: @@ -221,6 +242,10 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re } } + if d.EnableCompression { + req.Header.Set("Sec-Websocket-Extensions", "permessage-deflate; server_no_context_takeover; client_no_context_takeover") + } + hostPort, hostNoPort := hostPortNoPort(u) var proxyURL *url.URL @@ -298,12 +323,8 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re } if u.Scheme == "https" { - cfg := d.TLSClientConfig - if cfg == nil { - cfg = &tls.Config{ServerName: hostNoPort} - } else if cfg.ServerName == "" { - shallowCopy := *cfg - cfg = &shallowCopy + cfg := cloneTLSConfig(d.TLSClientConfig) + if cfg.ServerName == "" { cfg.ServerName = hostNoPort } tlsConn := tls.Client(netConn, cfg) @@ -328,6 +349,13 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re if err != nil { return nil, nil, err } + + if d.Jar != nil { + if rc := resp.Cookies(); len(rc) > 0 { + d.Jar.SetCookies(u, rc) + } + } + if resp.StatusCode != 101 || !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || @@ -341,6 +369,20 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re return nil, resp, ErrBadHandshake } + for _, ext := range parseExtensions(resp.Header) { + if ext[""] != "permessage-deflate" { + continue + } + _, snct := ext["server_no_context_takeover"] + _, cnct := ext["client_no_context_takeover"] + if !snct || !cnct { + return nil, resp, errInvalidCompression + } + conn.newCompressionWriter = compressNoContextTakeover + conn.newDecompressionReader = decompressNoContextTakeover + break + } + resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{})) conn.subprotocol = resp.Header.Get("Sec-Websocket-Protocol") diff --git a/vendor/github.com/gorilla/websocket/client_clone.go b/vendor/github.com/gorilla/websocket/client_clone.go new file mode 100644 index 00000000..4f0d9437 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_clone.go @@ -0,0 +1,16 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.8 + +package websocket + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return cfg.Clone() +} diff --git a/vendor/github.com/gorilla/websocket/client_clone_legacy.go b/vendor/github.com/gorilla/websocket/client_clone_legacy.go new file mode 100644 index 00000000..babb007f --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_clone_legacy.go @@ -0,0 +1,38 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.8 + +package websocket + +import "crypto/tls" + +// cloneTLSConfig clones all public fields except the fields +// SessionTicketsDisabled and SessionTicketKey. This avoids copying the +// sync.Mutex in the sync.Once and makes it safe to call cloneTLSConfig on a +// config in active use. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + } +} diff --git a/vendor/github.com/gorilla/websocket/client_server_test.go b/vendor/github.com/gorilla/websocket/client_server_test.go deleted file mode 100644 index 3f7345dd..00000000 --- a/vendor/github.com/gorilla/websocket/client_server_test.go +++ /dev/null @@ -1,451 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket - -import ( - "crypto/tls" - "crypto/x509" - "encoding/base64" - "io" - "io/ioutil" - "net" - "net/http" - "net/http/httptest" - "net/url" - "reflect" - "strings" - "testing" - "time" -) - -var cstUpgrader = Upgrader{ - Subprotocols: []string{"p0", "p1"}, - ReadBufferSize: 1024, - WriteBufferSize: 1024, - Error: func(w http.ResponseWriter, r *http.Request, status int, reason error) { - http.Error(w, reason.Error(), status) - }, -} - -var cstDialer = Dialer{ - Subprotocols: []string{"p1", "p2"}, - ReadBufferSize: 1024, - WriteBufferSize: 1024, -} - -type cstHandler struct{ *testing.T } - -type cstServer struct { - *httptest.Server - URL string -} - -const ( - cstPath = "/a/b" - cstRawQuery = "x=y" - cstRequestURI = cstPath + "?" + cstRawQuery -) - -func newServer(t *testing.T) *cstServer { - var s cstServer - s.Server = httptest.NewServer(cstHandler{t}) - s.Server.URL += cstRequestURI - s.URL = makeWsProto(s.Server.URL) - return &s -} - -func newTLSServer(t *testing.T) *cstServer { - var s cstServer - s.Server = httptest.NewTLSServer(cstHandler{t}) - s.Server.URL += cstRequestURI - s.URL = makeWsProto(s.Server.URL) - return &s -} - -func (t cstHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != cstPath { - t.Logf("path=%v, want %v", r.URL.Path, cstPath) - http.Error(w, "bad path", 400) - return - } - if r.URL.RawQuery != cstRawQuery { - t.Logf("query=%v, want %v", r.URL.RawQuery, cstRawQuery) - http.Error(w, "bad path", 400) - return - } - subprotos := Subprotocols(r) - if !reflect.DeepEqual(subprotos, cstDialer.Subprotocols) { - t.Logf("subprotols=%v, want %v", subprotos, cstDialer.Subprotocols) - http.Error(w, "bad protocol", 400) - return - } - ws, err := cstUpgrader.Upgrade(w, r, http.Header{"Set-Cookie": {"sessionID=1234"}}) - if err != nil { - t.Logf("Upgrade: %v", err) - return - } - defer ws.Close() - - if ws.Subprotocol() != "p1" { - t.Logf("Subprotocol() = %s, want p1", ws.Subprotocol()) - ws.Close() - return - } - op, rd, err := ws.NextReader() - if err != nil { - t.Logf("NextReader: %v", err) - return - } - wr, err := ws.NextWriter(op) - if err != nil { - t.Logf("NextWriter: %v", err) - return - } - if _, err = io.Copy(wr, rd); err != nil { - t.Logf("NextWriter: %v", err) - return - } - if err := wr.Close(); err != nil { - t.Logf("Close: %v", err) - return - } -} - -func makeWsProto(s string) string { - return "ws" + strings.TrimPrefix(s, "http") -} - -func sendRecv(t *testing.T, ws *Conn) { - const message = "Hello World!" - if err := ws.SetWriteDeadline(time.Now().Add(time.Second)); err != nil { - t.Fatalf("SetWriteDeadline: %v", err) - } - if err := ws.WriteMessage(TextMessage, []byte(message)); err != nil { - t.Fatalf("WriteMessage: %v", err) - } - if err := ws.SetReadDeadline(time.Now().Add(time.Second)); err != nil { - t.Fatalf("SetReadDeadline: %v", err) - } - _, p, err := ws.ReadMessage() - if err != nil { - t.Fatalf("ReadMessage: %v", err) - } - if string(p) != message { - t.Fatalf("message=%s, want %s", p, message) - } -} - -func TestProxyDial(t *testing.T) { - - s := newServer(t) - defer s.Close() - - surl, _ := url.Parse(s.URL) - - cstDialer.Proxy = http.ProxyURL(surl) - - connect := false - origHandler := s.Server.Config.Handler - - // Capture the request Host header. - s.Server.Config.Handler = http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - if r.Method == "CONNECT" { - connect = true - w.WriteHeader(200) - return - } - - if !connect { - t.Log("connect not recieved") - http.Error(w, "connect not recieved", 405) - return - } - origHandler.ServeHTTP(w, r) - }) - - ws, _, err := cstDialer.Dial(s.URL, nil) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer ws.Close() - sendRecv(t, ws) - - cstDialer.Proxy = http.ProxyFromEnvironment -} - -func TestProxyAuthorizationDial(t *testing.T) { - s := newServer(t) - defer s.Close() - - surl, _ := url.Parse(s.URL) - surl.User = url.UserPassword("username", "password") - cstDialer.Proxy = http.ProxyURL(surl) - - connect := false - origHandler := s.Server.Config.Handler - - // Capture the request Host header. - s.Server.Config.Handler = http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - proxyAuth := r.Header.Get("Proxy-Authorization") - expectedProxyAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte("username:password")) - if r.Method == "CONNECT" && proxyAuth == expectedProxyAuth { - connect = true - w.WriteHeader(200) - return - } - - if !connect { - t.Log("connect with proxy authorization not recieved") - http.Error(w, "connect with proxy authorization not recieved", 405) - return - } - origHandler.ServeHTTP(w, r) - }) - - ws, _, err := cstDialer.Dial(s.URL, nil) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer ws.Close() - sendRecv(t, ws) - - cstDialer.Proxy = http.ProxyFromEnvironment -} - -func TestDial(t *testing.T) { - s := newServer(t) - defer s.Close() - - ws, _, err := cstDialer.Dial(s.URL, nil) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer ws.Close() - sendRecv(t, ws) -} - -func TestDialTLS(t *testing.T) { - s := newTLSServer(t) - defer s.Close() - - certs := x509.NewCertPool() - for _, c := range s.TLS.Certificates { - roots, err := x509.ParseCertificates(c.Certificate[len(c.Certificate)-1]) - if err != nil { - t.Fatalf("error parsing server's root cert: %v", err) - } - for _, root := range roots { - certs.AddCert(root) - } - } - - u, _ := url.Parse(s.URL) - d := cstDialer - d.NetDial = func(network, addr string) (net.Conn, error) { return net.Dial(network, u.Host) } - d.TLSClientConfig = &tls.Config{RootCAs: certs} - ws, _, err := d.Dial("wss://example.com"+cstRequestURI, nil) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer ws.Close() - sendRecv(t, ws) -} - -func xTestDialTLSBadCert(t *testing.T) { - // This test is deactivated because of noisy logging from the net/http package. - s := newTLSServer(t) - defer s.Close() - - ws, _, err := cstDialer.Dial(s.URL, nil) - if err == nil { - ws.Close() - t.Fatalf("Dial: nil") - } -} - -func xTestDialTLSNoVerify(t *testing.T) { - s := newTLSServer(t) - defer s.Close() - - d := cstDialer - d.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - ws, _, err := d.Dial(s.URL, nil) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer ws.Close() - sendRecv(t, ws) -} - -func TestDialTimeout(t *testing.T) { - s := newServer(t) - defer s.Close() - - d := cstDialer - d.HandshakeTimeout = -1 - ws, _, err := d.Dial(s.URL, nil) - if err == nil { - ws.Close() - t.Fatalf("Dial: nil") - } -} - -func TestDialBadScheme(t *testing.T) { - s := newServer(t) - defer s.Close() - - ws, _, err := cstDialer.Dial(s.Server.URL, nil) - if err == nil { - ws.Close() - t.Fatalf("Dial: nil") - } -} - -func TestDialBadOrigin(t *testing.T) { - s := newServer(t) - defer s.Close() - - ws, resp, err := cstDialer.Dial(s.URL, http.Header{"Origin": {"bad"}}) - if err == nil { - ws.Close() - t.Fatalf("Dial: nil") - } - if resp == nil { - t.Fatalf("resp=nil, err=%v", err) - } - if resp.StatusCode != http.StatusForbidden { - t.Fatalf("status=%d, want %d", resp.StatusCode, http.StatusForbidden) - } -} - -func TestDialBadHeader(t *testing.T) { - s := newServer(t) - defer s.Close() - - for _, k := range []string{"Upgrade", - "Connection", - "Sec-Websocket-Key", - "Sec-Websocket-Version", - "Sec-Websocket-Protocol"} { - h := http.Header{} - h.Set(k, "bad") - ws, _, err := cstDialer.Dial(s.URL, http.Header{"Origin": {"bad"}}) - if err == nil { - ws.Close() - t.Errorf("Dial with header %s returned nil", k) - } - } -} - -func TestBadMethod(t *testing.T) { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ws, err := cstUpgrader.Upgrade(w, r, nil) - if err == nil { - t.Errorf("handshake succeeded, expect fail") - ws.Close() - } - })) - defer s.Close() - - resp, err := http.PostForm(s.URL, url.Values{}) - if err != nil { - t.Fatalf("PostForm returned error %v", err) - } - resp.Body.Close() - if resp.StatusCode != http.StatusMethodNotAllowed { - t.Errorf("Status = %d, want %d", resp.StatusCode, http.StatusMethodNotAllowed) - } -} - -func TestHandshake(t *testing.T) { - s := newServer(t) - defer s.Close() - - ws, resp, err := cstDialer.Dial(s.URL, http.Header{"Origin": {s.URL}}) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer ws.Close() - - var sessionID string - for _, c := range resp.Cookies() { - if c.Name == "sessionID" { - sessionID = c.Value - } - } - if sessionID != "1234" { - t.Error("Set-Cookie not received from the server.") - } - - if ws.Subprotocol() != "p1" { - t.Errorf("ws.Subprotocol() = %s, want p1", ws.Subprotocol()) - } - sendRecv(t, ws) -} - -func TestRespOnBadHandshake(t *testing.T) { - const expectedStatus = http.StatusGone - const expectedBody = "This is the response body." - - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(expectedStatus) - io.WriteString(w, expectedBody) - })) - defer s.Close() - - ws, resp, err := cstDialer.Dial(makeWsProto(s.URL), nil) - if err == nil { - ws.Close() - t.Fatalf("Dial: nil") - } - - if resp == nil { - t.Fatalf("resp=nil, err=%v", err) - } - - if resp.StatusCode != expectedStatus { - t.Errorf("resp.StatusCode=%d, want %d", resp.StatusCode, expectedStatus) - } - - p, err := ioutil.ReadAll(resp.Body) - if err != nil { - t.Fatalf("ReadFull(resp.Body) returned error %v", err) - } - - if string(p) != expectedBody { - t.Errorf("resp.Body=%s, want %s", p, expectedBody) - } -} - -// TestHostHeader confirms that the host header provided in the call to Dial is -// sent to the server. -func TestHostHeader(t *testing.T) { - s := newServer(t) - defer s.Close() - - specifiedHost := make(chan string, 1) - origHandler := s.Server.Config.Handler - - // Capture the request Host header. - s.Server.Config.Handler = http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - specifiedHost <- r.Host - origHandler.ServeHTTP(w, r) - }) - - ws, _, err := cstDialer.Dial(s.URL, http.Header{"Host": {"testhost"}}) - if err != nil { - t.Fatalf("Dial: %v", err) - } - defer ws.Close() - - if gotHost := <-specifiedHost; gotHost != "testhost" { - t.Fatalf("gotHost = %q, want \"testhost\"", gotHost) - } - - sendRecv(t, ws) -} diff --git a/vendor/github.com/gorilla/websocket/client_test.go b/vendor/github.com/gorilla/websocket/client_test.go deleted file mode 100644 index 7d2b0844..00000000 --- a/vendor/github.com/gorilla/websocket/client_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2014 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket - -import ( - "net/url" - "reflect" - "testing" -) - -var parseURLTests = []struct { - s string - u *url.URL - rui string -}{ - {"ws://example.com/", &url.URL{Scheme: "ws", Host: "example.com", Opaque: "/"}, "/"}, - {"ws://example.com", &url.URL{Scheme: "ws", Host: "example.com", Opaque: "/"}, "/"}, - {"ws://example.com:7777/", &url.URL{Scheme: "ws", Host: "example.com:7777", Opaque: "/"}, "/"}, - {"wss://example.com/", &url.URL{Scheme: "wss", Host: "example.com", Opaque: "/"}, "/"}, - {"wss://example.com/a/b", &url.URL{Scheme: "wss", Host: "example.com", Opaque: "/a/b"}, "/a/b"}, - {"ss://example.com/a/b", nil, ""}, - {"ws://webmaster@example.com/", nil, ""}, - {"wss://example.com/a/b?x=y", &url.URL{Scheme: "wss", Host: "example.com", Opaque: "/a/b", RawQuery: "x=y"}, "/a/b?x=y"}, - {"wss://example.com?x=y", &url.URL{Scheme: "wss", Host: "example.com", Opaque: "/", RawQuery: "x=y"}, "/?x=y"}, -} - -func TestParseURL(t *testing.T) { - for _, tt := range parseURLTests { - u, err := parseURL(tt.s) - if tt.u != nil && err != nil { - t.Errorf("parseURL(%q) returned error %v", tt.s, err) - continue - } - if tt.u == nil { - if err == nil { - t.Errorf("parseURL(%q) did not return error", tt.s) - } - continue - } - if !reflect.DeepEqual(u, tt.u) { - t.Errorf("parseURL(%q) = %v, want %v", tt.s, u, tt.u) - continue - } - if u.RequestURI() != tt.rui { - t.Errorf("parseURL(%q).RequestURI() = %v, want %v", tt.s, u.RequestURI(), tt.rui) - } - } -} - -var hostPortNoPortTests = []struct { - u *url.URL - hostPort, hostNoPort string -}{ - {&url.URL{Scheme: "ws", Host: "example.com"}, "example.com:80", "example.com"}, - {&url.URL{Scheme: "wss", Host: "example.com"}, "example.com:443", "example.com"}, - {&url.URL{Scheme: "ws", Host: "example.com:7777"}, "example.com:7777", "example.com"}, - {&url.URL{Scheme: "wss", Host: "example.com:7777"}, "example.com:7777", "example.com"}, -} - -func TestHostPortNoPort(t *testing.T) { - for _, tt := range hostPortNoPortTests { - hostPort, hostNoPort := hostPortNoPort(tt.u) - if hostPort != tt.hostPort { - t.Errorf("hostPortNoPort(%v) returned hostPort %q, want %q", tt.u, hostPort, tt.hostPort) - } - if hostNoPort != tt.hostNoPort { - t.Errorf("hostPortNoPort(%v) returned hostNoPort %q, want %q", tt.u, hostNoPort, tt.hostNoPort) - } - } -} diff --git a/vendor/github.com/gorilla/websocket/compression.go b/vendor/github.com/gorilla/websocket/compression.go new file mode 100644 index 00000000..813ffb1e --- /dev/null +++ b/vendor/github.com/gorilla/websocket/compression.go @@ -0,0 +1,148 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "compress/flate" + "errors" + "io" + "strings" + "sync" +) + +const ( + minCompressionLevel = -2 // flate.HuffmanOnly not defined in Go < 1.6 + maxCompressionLevel = flate.BestCompression + defaultCompressionLevel = 1 +) + +var ( + flateWriterPools [maxCompressionLevel - minCompressionLevel + 1]sync.Pool + flateReaderPool = sync.Pool{New: func() interface{} { + return flate.NewReader(nil) + }} +) + +func decompressNoContextTakeover(r io.Reader) io.ReadCloser { + const tail = + // Add four bytes as specified in RFC + "\x00\x00\xff\xff" + + // Add final block to squelch unexpected EOF error from flate reader. + "\x01\x00\x00\xff\xff" + + fr, _ := flateReaderPool.Get().(io.ReadCloser) + fr.(flate.Resetter).Reset(io.MultiReader(r, strings.NewReader(tail)), nil) + return &flateReadWrapper{fr} +} + +func isValidCompressionLevel(level int) bool { + return minCompressionLevel <= level && level <= maxCompressionLevel +} + +func compressNoContextTakeover(w io.WriteCloser, level int) io.WriteCloser { + p := &flateWriterPools[level-minCompressionLevel] + tw := &truncWriter{w: w} + fw, _ := p.Get().(*flate.Writer) + if fw == nil { + fw, _ = flate.NewWriter(tw, level) + } else { + fw.Reset(tw) + } + return &flateWriteWrapper{fw: fw, tw: tw, p: p} +} + +// truncWriter is an io.Writer that writes all but the last four bytes of the +// stream to another io.Writer. +type truncWriter struct { + w io.WriteCloser + n int + p [4]byte +} + +func (w *truncWriter) Write(p []byte) (int, error) { + n := 0 + + // fill buffer first for simplicity. + if w.n < len(w.p) { + n = copy(w.p[w.n:], p) + p = p[n:] + w.n += n + if len(p) == 0 { + return n, nil + } + } + + m := len(p) + if m > len(w.p) { + m = len(w.p) + } + + if nn, err := w.w.Write(w.p[:m]); err != nil { + return n + nn, err + } + + copy(w.p[:], w.p[m:]) + copy(w.p[len(w.p)-m:], p[len(p)-m:]) + nn, err := w.w.Write(p[:len(p)-m]) + return n + nn, err +} + +type flateWriteWrapper struct { + fw *flate.Writer + tw *truncWriter + p *sync.Pool +} + +func (w *flateWriteWrapper) Write(p []byte) (int, error) { + if w.fw == nil { + return 0, errWriteClosed + } + return w.fw.Write(p) +} + +func (w *flateWriteWrapper) Close() error { + if w.fw == nil { + return errWriteClosed + } + err1 := w.fw.Flush() + w.p.Put(w.fw) + w.fw = nil + if w.tw.p != [4]byte{0, 0, 0xff, 0xff} { + return errors.New("websocket: internal error, unexpected bytes at end of flate stream") + } + err2 := w.tw.w.Close() + if err1 != nil { + return err1 + } + return err2 +} + +type flateReadWrapper struct { + fr io.ReadCloser +} + +func (r *flateReadWrapper) Read(p []byte) (int, error) { + if r.fr == nil { + return 0, io.ErrClosedPipe + } + n, err := r.fr.Read(p) + if err == io.EOF { + // Preemptively place the reader back in the pool. This helps with + // scenarios where the application does not call NextReader() soon after + // this final read. + r.Close() + } + return n, err +} + +func (r *flateReadWrapper) Close() error { + if r.fr == nil { + return io.ErrClosedPipe + } + err := r.fr.Close() + flateReaderPool.Put(r.fr) + r.fr = nil + return err +} diff --git a/vendor/github.com/gorilla/websocket/conn.go b/vendor/github.com/gorilla/websocket/conn.go index eff26c63..97e1dbac 100644 --- a/vendor/github.com/gorilla/websocket/conn.go +++ b/vendor/github.com/gorilla/websocket/conn.go @@ -13,15 +13,25 @@ import ( "math/rand" "net" "strconv" + "sync" "time" + "unicode/utf8" ) const ( + // Frame header byte 0 bits from Section 5.2 of RFC 6455 + finalBit = 1 << 7 + rsv1Bit = 1 << 6 + rsv2Bit = 1 << 5 + rsv3Bit = 1 << 4 + + // Frame header byte 1 bits from Section 5.2 of RFC 6455 + maskBit = 1 << 7 + maxFrameHeaderSize = 2 + 8 + 4 // Fixed header + length + mask maxControlFramePayloadSize = 125 - finalBit = 1 << 7 - maskBit = 1 << 7 - writeWait = time.Second + + writeWait = time.Second defaultReadBufferSize = 4096 defaultWriteBufferSize = 4096 @@ -43,6 +53,8 @@ const ( CloseMessageTooBig = 1009 CloseMandatoryExtension = 1010 CloseInternalServerErr = 1011 + CloseServiceRestart = 1012 + CloseTryAgainLater = 1013 CloseTLSHandshake = 1015 ) @@ -169,6 +181,11 @@ var ( errInvalidControlFrame = errors.New("websocket: invalid control frame") ) +func newMaskKey() [4]byte { + n := rand.Uint32() + return [4]byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24)} +} + func hideTempErr(err error) error { if e, ok := err.(net.Error); ok && e.Temporary() { err = &netError{msg: e.Error(), timeout: e.Timeout()} @@ -184,74 +201,138 @@ func isData(frameType int) bool { return frameType == TextMessage || frameType == BinaryMessage } -func maskBytes(key [4]byte, pos int, b []byte) int { - for i := range b { - b[i] ^= key[pos&3] - pos++ - } - return pos & 3 +var validReceivedCloseCodes = map[int]bool{ + // see http://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number + + CloseNormalClosure: true, + CloseGoingAway: true, + CloseProtocolError: true, + CloseUnsupportedData: true, + CloseNoStatusReceived: false, + CloseAbnormalClosure: false, + CloseInvalidFramePayloadData: true, + ClosePolicyViolation: true, + CloseMessageTooBig: true, + CloseMandatoryExtension: true, + CloseInternalServerErr: true, + CloseServiceRestart: true, + CloseTryAgainLater: true, + CloseTLSHandshake: false, } -func newMaskKey() [4]byte { - n := rand.Uint32() - return [4]byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24)} +func isValidReceivedCloseCode(code int) bool { + return validReceivedCloseCodes[code] || (code >= 3000 && code <= 4999) } -// Conn represents a WebSocket connection. +// The Conn type represents a WebSocket connection. type Conn struct { conn net.Conn isServer bool subprotocol string // Write fields - mu chan bool // used as mutex to protect write to conn and closeSent - closeSent bool // true if close message was sent + mu chan bool // used as mutex to protect write to conn + writeBuf []byte // frame is constructed in this buffer. + writeDeadline time.Time + writer io.WriteCloser // the current writer returned to the application + isWriting bool // for best-effort concurrent write detection - // Message writer fields. - writeErr error - writeBuf []byte // frame is constructed in this buffer. - writePos int // end of data in writeBuf. - writeFrameType int // type of the current frame. - writeSeq int // incremented to invalidate message writers. - writeDeadline time.Time - isWriting bool // for best-effort concurrent write detection + writeErrMu sync.Mutex + writeErr error + + enableWriteCompression bool + compressionLevel int + newCompressionWriter func(io.WriteCloser, int) io.WriteCloser // Read fields + reader io.ReadCloser // the current reader returned to the application readErr error br *bufio.Reader readRemaining int64 // bytes remaining in current frame. readFinal bool // true the current message has more frames. - readSeq int // incremented to invalidate message readers. readLength int64 // Message size. readLimit int64 // Maximum message size. readMaskPos int readMaskKey [4]byte handlePong func(string) error handlePing func(string) error + handleClose func(int, string) error readErrCount int + messageReader *messageReader // the current low-level reader + + readDecompress bool // whether last read frame had RSV1 set + newDecompressionReader func(io.Reader) io.ReadCloser } func newConn(conn net.Conn, isServer bool, readBufferSize, writeBufferSize int) *Conn { + return newConnBRW(conn, isServer, readBufferSize, writeBufferSize, nil) +} + +type writeHook struct { + p []byte +} + +func (wh *writeHook) Write(p []byte) (int, error) { + wh.p = p + return len(p), nil +} + +func newConnBRW(conn net.Conn, isServer bool, readBufferSize, writeBufferSize int, brw *bufio.ReadWriter) *Conn { mu := make(chan bool, 1) mu <- true - if readBufferSize == 0 { - readBufferSize = defaultReadBufferSize + var br *bufio.Reader + if readBufferSize == 0 && brw != nil && brw.Reader != nil { + // Reuse the supplied bufio.Reader if the buffer has a useful size. + // This code assumes that peek on a reader returns + // bufio.Reader.buf[:0]. + brw.Reader.Reset(conn) + if p, err := brw.Reader.Peek(0); err == nil && cap(p) >= 256 { + br = brw.Reader + } } - if writeBufferSize == 0 { - writeBufferSize = defaultWriteBufferSize + if br == nil { + if readBufferSize == 0 { + readBufferSize = defaultReadBufferSize + } + if readBufferSize < maxControlFramePayloadSize { + readBufferSize = maxControlFramePayloadSize + } + br = bufio.NewReaderSize(conn, readBufferSize) + } + + var writeBuf []byte + if writeBufferSize == 0 && brw != nil && brw.Writer != nil { + // Use the bufio.Writer's buffer if the buffer has a useful size. This + // code assumes that bufio.Writer.buf[:1] is passed to the + // bufio.Writer's underlying writer. + var wh writeHook + brw.Writer.Reset(&wh) + brw.Writer.WriteByte(0) + brw.Flush() + if cap(wh.p) >= maxFrameHeaderSize+256 { + writeBuf = wh.p[:cap(wh.p)] + } + } + + if writeBuf == nil { + if writeBufferSize == 0 { + writeBufferSize = defaultWriteBufferSize + } + writeBuf = make([]byte, writeBufferSize+maxFrameHeaderSize) } c := &Conn{ - isServer: isServer, - br: bufio.NewReaderSize(conn, readBufferSize), - conn: conn, - mu: mu, - readFinal: true, - writeBuf: make([]byte, writeBufferSize+maxFrameHeaderSize), - writeFrameType: noFrame, - writePos: maxFrameHeaderSize, + isServer: isServer, + br: br, + conn: conn, + mu: mu, + readFinal: true, + writeBuf: writeBuf, + enableWriteCompression: true, + compressionLevel: defaultCompressionLevel, } + c.SetCloseHandler(nil) c.SetPingHandler(nil) c.SetPongHandler(nil) return c @@ -279,29 +360,40 @@ func (c *Conn) RemoteAddr() net.Addr { // Write methods +func (c *Conn) writeFatal(err error) error { + err = hideTempErr(err) + c.writeErrMu.Lock() + if c.writeErr == nil { + c.writeErr = err + } + c.writeErrMu.Unlock() + return err +} + func (c *Conn) write(frameType int, deadline time.Time, bufs ...[]byte) error { <-c.mu defer func() { c.mu <- true }() - if c.closeSent { - return ErrCloseSent - } else if frameType == CloseMessage { - c.closeSent = true + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err } c.conn.SetWriteDeadline(deadline) for _, buf := range bufs { if len(buf) > 0 { - n, err := c.conn.Write(buf) - if n != len(buf) { - // Close on partial write. - c.conn.Close() - } + _, err := c.conn.Write(buf) if err != nil { - return err + return c.writeFatal(err) } } } + + if frameType == CloseMessage { + c.writeFatal(ErrCloseSent) + } return nil } @@ -350,60 +442,104 @@ func (c *Conn) WriteControl(messageType int, data []byte, deadline time.Time) er } defer func() { c.mu <- true }() - if c.closeSent { - return ErrCloseSent - } else if messageType == CloseMessage { - c.closeSent = true + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err } c.conn.SetWriteDeadline(deadline) - n, err := c.conn.Write(buf) - if n != 0 && n != len(buf) { - c.conn.Close() + _, err = c.conn.Write(buf) + if err != nil { + return c.writeFatal(err) } - return hideTempErr(err) + if messageType == CloseMessage { + c.writeFatal(ErrCloseSent) + } + return err } -// NextWriter returns a writer for the next message to send. The writer's -// Close method flushes the complete message to the network. +func (c *Conn) prepWrite(messageType int) error { + // Close previous writer if not already closed by the application. It's + // probably better to return an error in this situation, but we cannot + // change this without breaking existing applications. + if c.writer != nil { + c.writer.Close() + c.writer = nil + } + + if !isControl(messageType) && !isData(messageType) { + return errBadWriteOpCode + } + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + return err +} + +// NextWriter returns a writer for the next message to send. The writer's Close +// method flushes the complete message to the network. // // There can be at most one open writer on a connection. NextWriter closes the // previous writer if the application has not already done so. func (c *Conn) NextWriter(messageType int) (io.WriteCloser, error) { - if c.writeErr != nil { - return nil, c.writeErr + if err := c.prepWrite(messageType); err != nil { + return nil, err } - if c.writeFrameType != noFrame { - if err := c.flushFrame(true, nil); err != nil { - return nil, err - } + mw := &messageWriter{ + c: c, + frameType: messageType, + pos: maxFrameHeaderSize, } - - if !isControl(messageType) && !isData(messageType) { - return nil, errBadWriteOpCode + c.writer = mw + if c.newCompressionWriter != nil && c.enableWriteCompression && isData(messageType) { + w := c.newCompressionWriter(c.writer, c.compressionLevel) + mw.compress = true + c.writer = w } - - c.writeFrameType = messageType - return messageWriter{c, c.writeSeq}, nil + return c.writer, nil } -func (c *Conn) flushFrame(final bool, extra []byte) error { - length := c.writePos - maxFrameHeaderSize + len(extra) +type messageWriter struct { + c *Conn + compress bool // whether next call to flushFrame should set RSV1 + pos int // end of data in writeBuf. + frameType int // type of the current frame. + err error +} + +func (w *messageWriter) fatal(err error) error { + if w.err != nil { + w.err = err + w.c.writer = nil + } + return err +} + +// flushFrame writes buffered data and extra as a frame to the network. The +// final argument indicates that this is the last frame in the message. +func (w *messageWriter) flushFrame(final bool, extra []byte) error { + c := w.c + length := w.pos - maxFrameHeaderSize + len(extra) // Check for invalid control frames. - if isControl(c.writeFrameType) && + if isControl(w.frameType) && (!final || length > maxControlFramePayloadSize) { - c.writeSeq++ - c.writeFrameType = noFrame - c.writePos = maxFrameHeaderSize - return errInvalidControlFrame + return w.fatal(errInvalidControlFrame) } - b0 := byte(c.writeFrameType) + b0 := byte(w.frameType) if final { b0 |= finalBit } + if w.compress { + b0 |= rsv1Bit + } + w.compress = false + b1 := byte(0) if !c.isServer { b1 |= maskBit @@ -435,10 +571,9 @@ func (c *Conn) flushFrame(final bool, extra []byte) error { if !c.isServer { key := newMaskKey() copy(c.writeBuf[maxFrameHeaderSize-4:], key[:]) - maskBytes(key, 0, c.writeBuf[maxFrameHeaderSize:c.writePos]) + maskBytes(key, 0, c.writeBuf[maxFrameHeaderSize:w.pos]) if len(extra) > 0 { - c.writeErr = errors.New("websocket: internal error, extra used in client mode") - return c.writeErr + return c.writeFatal(errors.New("websocket: internal error, extra used in client mode")) } } @@ -451,46 +586,35 @@ func (c *Conn) flushFrame(final bool, extra []byte) error { } c.isWriting = true - c.writeErr = c.write(c.writeFrameType, c.writeDeadline, c.writeBuf[framePos:c.writePos], extra) + err := c.write(w.frameType, c.writeDeadline, c.writeBuf[framePos:w.pos], extra) if !c.isWriting { panic("concurrent write to websocket connection") } c.isWriting = false - // Setup for next frame. - c.writePos = maxFrameHeaderSize - c.writeFrameType = continuationFrame + if err != nil { + return w.fatal(err) + } + if final { - c.writeSeq++ - c.writeFrameType = noFrame + c.writer = nil + return nil } - return c.writeErr -} -type messageWriter struct { - c *Conn - seq int -} - -func (w messageWriter) err() error { - c := w.c - if c.writeSeq != w.seq { - return errWriteClosed - } - if c.writeErr != nil { - return c.writeErr - } + // Setup for next frame. + w.pos = maxFrameHeaderSize + w.frameType = continuationFrame return nil } -func (w messageWriter) ncopy(max int) (int, error) { - n := len(w.c.writeBuf) - w.c.writePos +func (w *messageWriter) ncopy(max int) (int, error) { + n := len(w.c.writeBuf) - w.pos if n <= 0 { - if err := w.c.flushFrame(false, nil); err != nil { + if err := w.flushFrame(false, nil); err != nil { return 0, err } - n = len(w.c.writeBuf) - w.c.writePos + n = len(w.c.writeBuf) - w.pos } if n > max { n = max @@ -498,14 +622,14 @@ func (w messageWriter) ncopy(max int) (int, error) { return n, nil } -func (w messageWriter) write(final bool, p []byte) (int, error) { - if err := w.err(); err != nil { - return 0, err +func (w *messageWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, w.err } if len(p) > 2*len(w.c.writeBuf) && w.c.isServer { // Don't buffer large messages. - err := w.c.flushFrame(final, p) + err := w.flushFrame(false, p) if err != nil { return 0, err } @@ -518,20 +642,16 @@ func (w messageWriter) write(final bool, p []byte) (int, error) { if err != nil { return 0, err } - copy(w.c.writeBuf[w.c.writePos:], p[:n]) - w.c.writePos += n + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n p = p[n:] } return nn, nil } -func (w messageWriter) Write(p []byte) (int, error) { - return w.write(false, p) -} - -func (w messageWriter) WriteString(p string) (int, error) { - if err := w.err(); err != nil { - return 0, err +func (w *messageWriter) WriteString(p string) (int, error) { + if w.err != nil { + return 0, w.err } nn := len(p) @@ -540,27 +660,27 @@ func (w messageWriter) WriteString(p string) (int, error) { if err != nil { return 0, err } - copy(w.c.writeBuf[w.c.writePos:], p[:n]) - w.c.writePos += n + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n p = p[n:] } return nn, nil } -func (w messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { - if err := w.err(); err != nil { - return 0, err +func (w *messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { + if w.err != nil { + return 0, w.err } for { - if w.c.writePos == len(w.c.writeBuf) { - err = w.c.flushFrame(false, nil) + if w.pos == len(w.c.writeBuf) { + err = w.flushFrame(false, nil) if err != nil { break } } var n int - n, err = r.Read(w.c.writeBuf[w.c.writePos:]) - w.c.writePos += n + n, err = r.Read(w.c.writeBuf[w.pos:]) + w.pos += n nn += int64(n) if err != nil { if err == io.EOF { @@ -572,30 +692,64 @@ func (w messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { return nn, err } -func (w messageWriter) Close() error { - if err := w.err(); err != nil { +func (w *messageWriter) Close() error { + if w.err != nil { + return w.err + } + if err := w.flushFrame(true, nil); err != nil { return err } - return w.c.flushFrame(true, nil) + w.err = errWriteClosed + return nil +} + +// WritePreparedMessage writes prepared message into connection. +func (c *Conn) WritePreparedMessage(pm *PreparedMessage) error { + frameType, frameData, err := pm.frame(prepareKey{ + isServer: c.isServer, + compress: c.newCompressionWriter != nil && c.enableWriteCompression && isData(pm.messageType), + compressionLevel: c.compressionLevel, + }) + if err != nil { + return err + } + if c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = true + err = c.write(frameType, c.writeDeadline, frameData, nil) + if !c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = false + return err } // WriteMessage is a helper method for getting a writer using NextWriter, // writing the message and closing the writer. func (c *Conn) WriteMessage(messageType int, data []byte) error { - wr, err := c.NextWriter(messageType) + + if c.isServer && (c.newCompressionWriter == nil || !c.enableWriteCompression) { + // Fast path with no allocations and single frame. + + if err := c.prepWrite(messageType); err != nil { + return err + } + mw := messageWriter{c: c, frameType: messageType, pos: maxFrameHeaderSize} + n := copy(c.writeBuf[mw.pos:], data) + mw.pos += n + data = data[n:] + return mw.flushFrame(true, data) + } + + w, err := c.NextWriter(messageType) if err != nil { return err } - w := wr.(messageWriter) - if _, err := w.write(true, data); err != nil { + if _, err = w.Write(data); err != nil { return err } - if c.writeSeq == w.seq { - if err := c.flushFrame(true, nil); err != nil { - return err - } - } - return nil + return w.Close() } // SetWriteDeadline sets the write deadline on the underlying network @@ -609,22 +763,6 @@ func (c *Conn) SetWriteDeadline(t time.Time) error { // Read methods -// readFull is like io.ReadFull except that io.EOF is never returned. -func (c *Conn) readFull(p []byte) (err error) { - var n int - for n < len(p) && err == nil { - var nn int - nn, err = c.br.Read(p[n:]) - n += nn - } - if n == len(p) { - err = nil - } else if err == io.EOF { - err = errUnexpectedEOF - } - return -} - func (c *Conn) advanceFrame() (int, error) { // 1. Skip remainder of previous frame. @@ -637,19 +775,24 @@ func (c *Conn) advanceFrame() (int, error) { // 2. Read and parse first two bytes of frame header. - var b [8]byte - if err := c.readFull(b[:2]); err != nil { + p, err := c.read(2) + if err != nil { return noFrame, err } - final := b[0]&finalBit != 0 - frameType := int(b[0] & 0xf) - reserved := int((b[0] >> 4) & 0x7) - mask := b[1]&maskBit != 0 - c.readRemaining = int64(b[1] & 0x7f) + final := p[0]&finalBit != 0 + frameType := int(p[0] & 0xf) + mask := p[1]&maskBit != 0 + c.readRemaining = int64(p[1] & 0x7f) - if reserved != 0 { - return noFrame, c.handleProtocolError("unexpected reserved bits " + strconv.Itoa(reserved)) + c.readDecompress = false + if c.newDecompressionReader != nil && (p[0]&rsv1Bit) != 0 { + c.readDecompress = true + p[0] &^= rsv1Bit + } + + if rsv := p[0] & (rsv1Bit | rsv2Bit | rsv3Bit); rsv != 0 { + return noFrame, c.handleProtocolError("unexpected reserved bits 0x" + strconv.FormatInt(int64(rsv), 16)) } switch frameType { @@ -678,15 +821,17 @@ func (c *Conn) advanceFrame() (int, error) { switch c.readRemaining { case 126: - if err := c.readFull(b[:2]); err != nil { + p, err := c.read(2) + if err != nil { return noFrame, err } - c.readRemaining = int64(binary.BigEndian.Uint16(b[:2])) + c.readRemaining = int64(binary.BigEndian.Uint16(p)) case 127: - if err := c.readFull(b[:8]); err != nil { + p, err := c.read(8) + if err != nil { return noFrame, err } - c.readRemaining = int64(binary.BigEndian.Uint64(b[:8])) + c.readRemaining = int64(binary.BigEndian.Uint64(p)) } // 4. Handle frame masking. @@ -697,9 +842,11 @@ func (c *Conn) advanceFrame() (int, error) { if mask { c.readMaskPos = 0 - if err := c.readFull(c.readMaskKey[:]); err != nil { + p, err := c.read(len(c.readMaskKey)) + if err != nil { return noFrame, err } + copy(c.readMaskKey[:], p) } // 5. For text and binary messages, enforce read limit and return. @@ -719,9 +866,9 @@ func (c *Conn) advanceFrame() (int, error) { var payload []byte if c.readRemaining > 0 { - payload = make([]byte, c.readRemaining) + payload, err = c.read(int(c.readRemaining)) c.readRemaining = 0 - if err := c.readFull(payload); err != nil { + if err != nil { return noFrame, err } if c.isServer { @@ -741,15 +888,21 @@ func (c *Conn) advanceFrame() (int, error) { return noFrame, err } case CloseMessage: - echoMessage := []byte{} closeCode := CloseNoStatusReceived closeText := "" if len(payload) >= 2 { - echoMessage = payload[:2] closeCode = int(binary.BigEndian.Uint16(payload)) + if !isValidReceivedCloseCode(closeCode) { + return noFrame, c.handleProtocolError("invalid close code") + } closeText = string(payload[2:]) + if !utf8.ValidString(closeText) { + return noFrame, c.handleProtocolError("invalid utf8 payload in close frame") + } + } + if err := c.handleClose(closeCode, closeText); err != nil { + return noFrame, err } - c.WriteControl(CloseMessage, echoMessage, time.Now().Add(writeWait)) return noFrame, &CloseError{Code: closeCode, Text: closeText} } @@ -772,8 +925,13 @@ func (c *Conn) handleProtocolError(message string) error { // permanent. Once this method returns a non-nil error, all subsequent calls to // this method return the same error. func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { + // Close previous reader, only relevant for decompression. + if c.reader != nil { + c.reader.Close() + c.reader = nil + } - c.readSeq++ + c.messageReader = nil c.readLength = 0 for c.readErr == nil { @@ -783,7 +941,12 @@ func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { break } if frameType == TextMessage || frameType == BinaryMessage { - return frameType, messageReader{c, c.readSeq}, nil + c.messageReader = &messageReader{c} + c.reader = c.messageReader + if c.readDecompress { + c.reader = c.newDecompressionReader(c.reader) + } + return frameType, c.reader, nil } } @@ -798,53 +961,57 @@ func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { return noFrame, nil, c.readErr } -type messageReader struct { - c *Conn - seq int -} +type messageReader struct{ c *Conn } -func (r messageReader) Read(b []byte) (int, error) { - - if r.seq != r.c.readSeq { +func (r *messageReader) Read(b []byte) (int, error) { + c := r.c + if c.messageReader != r { return 0, io.EOF } - for r.c.readErr == nil { + for c.readErr == nil { - if r.c.readRemaining > 0 { - if int64(len(b)) > r.c.readRemaining { - b = b[:r.c.readRemaining] + if c.readRemaining > 0 { + if int64(len(b)) > c.readRemaining { + b = b[:c.readRemaining] } - n, err := r.c.br.Read(b) - r.c.readErr = hideTempErr(err) - if r.c.isServer { - r.c.readMaskPos = maskBytes(r.c.readMaskKey, r.c.readMaskPos, b[:n]) + n, err := c.br.Read(b) + c.readErr = hideTempErr(err) + if c.isServer { + c.readMaskPos = maskBytes(c.readMaskKey, c.readMaskPos, b[:n]) } - r.c.readRemaining -= int64(n) - return n, r.c.readErr + c.readRemaining -= int64(n) + if c.readRemaining > 0 && c.readErr == io.EOF { + c.readErr = errUnexpectedEOF + } + return n, c.readErr } - if r.c.readFinal { - r.c.readSeq++ + if c.readFinal { + c.messageReader = nil return 0, io.EOF } - frameType, err := r.c.advanceFrame() + frameType, err := c.advanceFrame() switch { case err != nil: - r.c.readErr = hideTempErr(err) + c.readErr = hideTempErr(err) case frameType == TextMessage || frameType == BinaryMessage: - r.c.readErr = errors.New("websocket: internal error, unexpected text or binary in Reader") + c.readErr = errors.New("websocket: internal error, unexpected text or binary in Reader") } } - err := r.c.readErr - if err == io.EOF && r.seq == r.c.readSeq { + err := c.readErr + if err == io.EOF && c.messageReader == r { err = errUnexpectedEOF } return 0, err } +func (r *messageReader) Close() error { + return nil +} + // ReadMessage is a helper method for getting a reader using NextReader and // reading from that reader to a buffer. func (c *Conn) ReadMessage() (messageType int, p []byte, err error) { @@ -872,9 +1039,49 @@ func (c *Conn) SetReadLimit(limit int64) { c.readLimit = limit } +// CloseHandler returns the current close handler +func (c *Conn) CloseHandler() func(code int, text string) error { + return c.handleClose +} + +// SetCloseHandler sets the handler for close messages received from the peer. +// The code argument to h is the received close code or CloseNoStatusReceived +// if the close message is empty. The default close handler sends a close frame +// back to the peer. +// +// The application must read the connection to process close messages as +// described in the section on Control Frames above. +// +// The connection read methods return a CloseError when a close frame is +// received. Most applications should handle close messages as part of their +// normal error handling. Applications should only set a close handler when the +// application must perform some action before sending a close frame back to +// the peer. +func (c *Conn) SetCloseHandler(h func(code int, text string) error) { + if h == nil { + h = func(code int, text string) error { + message := []byte{} + if code != CloseNoStatusReceived { + message = FormatCloseMessage(code, "") + } + c.WriteControl(CloseMessage, message, time.Now().Add(writeWait)) + return nil + } + } + c.handleClose = h +} + +// PingHandler returns the current ping handler +func (c *Conn) PingHandler() func(appData string) error { + return c.handlePing +} + // SetPingHandler sets the handler for ping messages received from the peer. // The appData argument to h is the PING frame application data. The default // ping handler sends a pong to the peer. +// +// The application must read the connection to process ping messages as +// described in the section on Control Frames above. func (c *Conn) SetPingHandler(h func(appData string) error) { if h == nil { h = func(message string) error { @@ -890,9 +1097,17 @@ func (c *Conn) SetPingHandler(h func(appData string) error) { c.handlePing = h } +// PongHandler returns the current pong handler +func (c *Conn) PongHandler() func(appData string) error { + return c.handlePong +} + // SetPongHandler sets the handler for pong messages received from the peer. // The appData argument to h is the PONG frame application data. The default // pong handler does nothing. +// +// The application must read the connection to process ping messages as +// described in the section on Control Frames above. func (c *Conn) SetPongHandler(h func(appData string) error) { if h == nil { h = func(string) error { return nil } @@ -906,6 +1121,25 @@ func (c *Conn) UnderlyingConn() net.Conn { return c.conn } +// EnableWriteCompression enables and disables write compression of +// subsequent text and binary messages. This function is a noop if +// compression was not negotiated with the peer. +func (c *Conn) EnableWriteCompression(enable bool) { + c.enableWriteCompression = enable +} + +// SetCompressionLevel sets the flate compression level for subsequent text and +// binary messages. This function is a noop if compression was not negotiated +// with the peer. See the compress/flate package for a description of +// compression levels. +func (c *Conn) SetCompressionLevel(level int) error { + if !isValidCompressionLevel(level) { + return errors.New("websocket: invalid compression level") + } + c.compressionLevel = level + return nil +} + // FormatCloseMessage formats closeCode and text as a WebSocket close message. func FormatCloseMessage(closeCode int, text string) []byte { buf := make([]byte, 2+len(text)) diff --git a/vendor/github.com/gorilla/websocket/conn_read.go b/vendor/github.com/gorilla/websocket/conn_read.go new file mode 100644 index 00000000..1ea15059 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_read.go @@ -0,0 +1,18 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.5 + +package websocket + +import "io" + +func (c *Conn) read(n int) ([]byte, error) { + p, err := c.br.Peek(n) + if err == io.EOF { + err = errUnexpectedEOF + } + c.br.Discard(len(p)) + return p, err +} diff --git a/vendor/github.com/gorilla/websocket/conn_read_legacy.go b/vendor/github.com/gorilla/websocket/conn_read_legacy.go new file mode 100644 index 00000000..018541cf --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_read_legacy.go @@ -0,0 +1,21 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.5 + +package websocket + +import "io" + +func (c *Conn) read(n int) ([]byte, error) { + p, err := c.br.Peek(n) + if err == io.EOF { + err = errUnexpectedEOF + } + if len(p) > 0 { + // advance over the bytes just read + io.ReadFull(c.br, p) + } + return p, err +} diff --git a/vendor/github.com/gorilla/websocket/conn_test.go b/vendor/github.com/gorilla/websocket/conn_test.go deleted file mode 100644 index 04c8dd8d..00000000 --- a/vendor/github.com/gorilla/websocket/conn_test.go +++ /dev/null @@ -1,367 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "net" - "reflect" - "testing" - "testing/iotest" - "time" -) - -var _ net.Error = errWriteTimeout - -type fakeNetConn struct { - io.Reader - io.Writer -} - -func (c fakeNetConn) Close() error { return nil } -func (c fakeNetConn) LocalAddr() net.Addr { return nil } -func (c fakeNetConn) RemoteAddr() net.Addr { return nil } -func (c fakeNetConn) SetDeadline(t time.Time) error { return nil } -func (c fakeNetConn) SetReadDeadline(t time.Time) error { return nil } -func (c fakeNetConn) SetWriteDeadline(t time.Time) error { return nil } - -func TestFraming(t *testing.T) { - frameSizes := []int{0, 1, 2, 124, 125, 126, 127, 128, 129, 65534, 65535, 65536, 65537} - var readChunkers = []struct { - name string - f func(io.Reader) io.Reader - }{ - {"half", iotest.HalfReader}, - {"one", iotest.OneByteReader}, - {"asis", func(r io.Reader) io.Reader { return r }}, - } - - writeBuf := make([]byte, 65537) - for i := range writeBuf { - writeBuf[i] = byte(i) - } - - for _, isServer := range []bool{true, false} { - for _, chunker := range readChunkers { - - var connBuf bytes.Buffer - wc := newConn(fakeNetConn{Reader: nil, Writer: &connBuf}, isServer, 1024, 1024) - rc := newConn(fakeNetConn{Reader: chunker.f(&connBuf), Writer: nil}, !isServer, 1024, 1024) - - for _, n := range frameSizes { - for _, iocopy := range []bool{true, false} { - name := fmt.Sprintf("s:%v, r:%s, n:%d c:%v", isServer, chunker.name, n, iocopy) - - w, err := wc.NextWriter(TextMessage) - if err != nil { - t.Errorf("%s: wc.NextWriter() returned %v", name, err) - continue - } - var nn int - if iocopy { - var n64 int64 - n64, err = io.Copy(w, bytes.NewReader(writeBuf[:n])) - nn = int(n64) - } else { - nn, err = w.Write(writeBuf[:n]) - } - if err != nil || nn != n { - t.Errorf("%s: w.Write(writeBuf[:n]) returned %d, %v", name, nn, err) - continue - } - err = w.Close() - if err != nil { - t.Errorf("%s: w.Close() returned %v", name, err) - continue - } - - opCode, r, err := rc.NextReader() - if err != nil || opCode != TextMessage { - t.Errorf("%s: NextReader() returned %d, r, %v", name, opCode, err) - continue - } - rbuf, err := ioutil.ReadAll(r) - if err != nil { - t.Errorf("%s: ReadFull() returned rbuf, %v", name, err) - continue - } - - if len(rbuf) != n { - t.Errorf("%s: len(rbuf) is %d, want %d", name, len(rbuf), n) - continue - } - - for i, b := range rbuf { - if byte(i) != b { - t.Errorf("%s: bad byte at offset %d", name, i) - break - } - } - } - } - } - } -} - -func TestControl(t *testing.T) { - const message = "this is a ping/pong messsage" - for _, isServer := range []bool{true, false} { - for _, isWriteControl := range []bool{true, false} { - name := fmt.Sprintf("s:%v, wc:%v", isServer, isWriteControl) - var connBuf bytes.Buffer - wc := newConn(fakeNetConn{Reader: nil, Writer: &connBuf}, isServer, 1024, 1024) - rc := newConn(fakeNetConn{Reader: &connBuf, Writer: nil}, !isServer, 1024, 1024) - if isWriteControl { - wc.WriteControl(PongMessage, []byte(message), time.Now().Add(time.Second)) - } else { - w, err := wc.NextWriter(PongMessage) - if err != nil { - t.Errorf("%s: wc.NextWriter() returned %v", name, err) - continue - } - if _, err := w.Write([]byte(message)); err != nil { - t.Errorf("%s: w.Write() returned %v", name, err) - continue - } - if err := w.Close(); err != nil { - t.Errorf("%s: w.Close() returned %v", name, err) - continue - } - var actualMessage string - rc.SetPongHandler(func(s string) error { actualMessage = s; return nil }) - rc.NextReader() - if actualMessage != message { - t.Errorf("%s: pong=%q, want %q", name, actualMessage, message) - continue - } - } - } - } -} - -func TestCloseBeforeFinalFrame(t *testing.T) { - const bufSize = 512 - - expectedErr := &CloseError{Code: CloseNormalClosure, Text: "hello"} - - var b1, b2 bytes.Buffer - wc := newConn(fakeNetConn{Reader: nil, Writer: &b1}, false, 1024, bufSize) - rc := newConn(fakeNetConn{Reader: &b1, Writer: &b2}, true, 1024, 1024) - - w, _ := wc.NextWriter(BinaryMessage) - w.Write(make([]byte, bufSize+bufSize/2)) - wc.WriteControl(CloseMessage, FormatCloseMessage(expectedErr.Code, expectedErr.Text), time.Now().Add(10*time.Second)) - w.Close() - - op, r, err := rc.NextReader() - if op != BinaryMessage || err != nil { - t.Fatalf("NextReader() returned %d, %v", op, err) - } - _, err = io.Copy(ioutil.Discard, r) - if !reflect.DeepEqual(err, expectedErr) { - t.Fatalf("io.Copy() returned %v, want %v", err, expectedErr) - } - _, _, err = rc.NextReader() - if !reflect.DeepEqual(err, expectedErr) { - t.Fatalf("NextReader() returned %v, want %v", err, expectedErr) - } -} - -func TestEOFBeforeFinalFrame(t *testing.T) { - const bufSize = 512 - - var b1, b2 bytes.Buffer - wc := newConn(fakeNetConn{Reader: nil, Writer: &b1}, false, 1024, bufSize) - rc := newConn(fakeNetConn{Reader: &b1, Writer: &b2}, true, 1024, 1024) - - w, _ := wc.NextWriter(BinaryMessage) - w.Write(make([]byte, bufSize+bufSize/2)) - - op, r, err := rc.NextReader() - if op != BinaryMessage || err != nil { - t.Fatalf("NextReader() returned %d, %v", op, err) - } - _, err = io.Copy(ioutil.Discard, r) - if err != errUnexpectedEOF { - t.Fatalf("io.Copy() returned %v, want %v", err, errUnexpectedEOF) - } - _, _, err = rc.NextReader() - if err != errUnexpectedEOF { - t.Fatalf("NextReader() returned %v, want %v", err, errUnexpectedEOF) - } -} - -func TestReadLimit(t *testing.T) { - - const readLimit = 512 - message := make([]byte, readLimit+1) - - var b1, b2 bytes.Buffer - wc := newConn(fakeNetConn{Reader: nil, Writer: &b1}, false, 1024, readLimit-2) - rc := newConn(fakeNetConn{Reader: &b1, Writer: &b2}, true, 1024, 1024) - rc.SetReadLimit(readLimit) - - // Send message at the limit with interleaved pong. - w, _ := wc.NextWriter(BinaryMessage) - w.Write(message[:readLimit-1]) - wc.WriteControl(PongMessage, []byte("this is a pong"), time.Now().Add(10*time.Second)) - w.Write(message[:1]) - w.Close() - - // Send message larger than the limit. - wc.WriteMessage(BinaryMessage, message[:readLimit+1]) - - op, _, err := rc.NextReader() - if op != BinaryMessage || err != nil { - t.Fatalf("1: NextReader() returned %d, %v", op, err) - } - op, r, err := rc.NextReader() - if op != BinaryMessage || err != nil { - t.Fatalf("2: NextReader() returned %d, %v", op, err) - } - _, err = io.Copy(ioutil.Discard, r) - if err != ErrReadLimit { - t.Fatalf("io.Copy() returned %v", err) - } -} - -func TestUnderlyingConn(t *testing.T) { - var b1, b2 bytes.Buffer - fc := fakeNetConn{Reader: &b1, Writer: &b2} - c := newConn(fc, true, 1024, 1024) - ul := c.UnderlyingConn() - if ul != fc { - t.Fatalf("Underlying conn is not what it should be.") - } -} - -func TestBufioReadBytes(t *testing.T) { - - // Test calling bufio.ReadBytes for value longer than read buffer size. - - m := make([]byte, 512) - m[len(m)-1] = '\n' - - var b1, b2 bytes.Buffer - wc := newConn(fakeNetConn{Reader: nil, Writer: &b1}, false, len(m)+64, len(m)+64) - rc := newConn(fakeNetConn{Reader: &b1, Writer: &b2}, true, len(m)-64, len(m)-64) - - w, _ := wc.NextWriter(BinaryMessage) - w.Write(m) - w.Close() - - op, r, err := rc.NextReader() - if op != BinaryMessage || err != nil { - t.Fatalf("NextReader() returned %d, %v", op, err) - } - - br := bufio.NewReader(r) - p, err := br.ReadBytes('\n') - if err != nil { - t.Fatalf("ReadBytes() returned %v", err) - } - if len(p) != len(m) { - t.Fatalf("read returnd %d bytes, want %d bytes", len(p), len(m)) - } -} - -var closeErrorTests = []struct { - err error - codes []int - ok bool -}{ - {&CloseError{Code: CloseNormalClosure}, []int{CloseNormalClosure}, true}, - {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived}, false}, - {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived, CloseNormalClosure}, true}, - {errors.New("hello"), []int{CloseNormalClosure}, false}, -} - -func TestCloseError(t *testing.T) { - for _, tt := range closeErrorTests { - ok := IsCloseError(tt.err, tt.codes...) - if ok != tt.ok { - t.Errorf("IsCloseError(%#v, %#v) returned %v, want %v", tt.err, tt.codes, ok, tt.ok) - } - } -} - -var unexpectedCloseErrorTests = []struct { - err error - codes []int - ok bool -}{ - {&CloseError{Code: CloseNormalClosure}, []int{CloseNormalClosure}, false}, - {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived}, true}, - {&CloseError{Code: CloseNormalClosure}, []int{CloseNoStatusReceived, CloseNormalClosure}, false}, - {errors.New("hello"), []int{CloseNormalClosure}, false}, -} - -func TestUnexpectedCloseErrors(t *testing.T) { - for _, tt := range unexpectedCloseErrorTests { - ok := IsUnexpectedCloseError(tt.err, tt.codes...) - if ok != tt.ok { - t.Errorf("IsUnexpectedCloseError(%#v, %#v) returned %v, want %v", tt.err, tt.codes, ok, tt.ok) - } - } -} - -type blockingWriter struct { - c1, c2 chan struct{} -} - -func (w blockingWriter) Write(p []byte) (int, error) { - // Allow main to continue - close(w.c1) - // Wait for panic in main - <-w.c2 - return len(p), nil -} - -func TestConcurrentWritePanic(t *testing.T) { - w := blockingWriter{make(chan struct{}), make(chan struct{})} - c := newConn(fakeNetConn{Reader: nil, Writer: w}, false, 1024, 1024) - go func() { - c.WriteMessage(TextMessage, []byte{}) - }() - - // wait for goroutine to block in write. - <-w.c1 - - defer func() { - close(w.c2) - if v := recover(); v != nil { - return - } - }() - - c.WriteMessage(TextMessage, []byte{}) - t.Fatal("should not get here") -} - -type failingReader struct{} - -func (r failingReader) Read(p []byte) (int, error) { - return 0, io.EOF -} - -func TestFailedConnectionReadPanic(t *testing.T) { - c := newConn(fakeNetConn{Reader: failingReader{}, Writer: nil}, false, 1024, 1024) - - defer func() { - if v := recover(); v != nil { - return - } - }() - - for i := 0; i < 20000; i++ { - c.ReadMessage() - } - t.Fatal("should not get here") -} diff --git a/vendor/github.com/gorilla/websocket/doc.go b/vendor/github.com/gorilla/websocket/doc.go index 499b03db..e291a952 100644 --- a/vendor/github.com/gorilla/websocket/doc.go +++ b/vendor/github.com/gorilla/websocket/doc.go @@ -85,18 +85,22 @@ // and pong. Call the connection WriteControl, WriteMessage or NextWriter // methods to send a control message to the peer. // -// Connections handle received ping and pong messages by invoking callback -// functions set with SetPingHandler and SetPongHandler methods. The default -// ping handler sends a pong to the client. The callback functions can be -// invoked from the NextReader, ReadMessage or the message Read method. -// // Connections handle received close messages by sending a close message to the // peer and returning a *CloseError from the the NextReader, ReadMessage or the // message Read method. // -// The application must read the connection to process ping and close messages -// sent from the peer. If the application is not otherwise interested in -// messages from the peer, then the application should start a goroutine to +// Connections handle received ping and pong messages by invoking callback +// functions set with SetPingHandler and SetPongHandler methods. The callback +// functions are called from the NextReader, ReadMessage and the message Read +// methods. +// +// The default ping handler sends a pong to the peer. The application's reading +// goroutine can block for a short time while the handler writes the pong data +// to the connection. +// +// The application must read the connection to process ping, pong and close +// messages sent from the peer. If the application is not otherwise interested +// in messages from the peer, then the application should start a goroutine to // read and discard messages from the peer. A simple example is: // // func readLoop(c *websocket.Conn) { @@ -114,9 +118,10 @@ // // Applications are responsible for ensuring that no more than one goroutine // calls the write methods (NextWriter, SetWriteDeadline, WriteMessage, -// WriteJSON) concurrently and that no more than one goroutine calls the read -// methods (NextReader, SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, -// SetPingHandler) concurrently. +// WriteJSON, EnableWriteCompression, SetCompressionLevel) concurrently and +// that no more than one goroutine calls the read methods (NextReader, +// SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, SetPingHandler) +// concurrently. // // The Close and WriteControl methods can be called concurrently with all other // methods. @@ -145,4 +150,31 @@ // The deprecated Upgrade function does not enforce an origin policy. It's the // application's responsibility to check the Origin header before calling // Upgrade. +// +// Compression EXPERIMENTAL +// +// Per message compression extensions (RFC 7692) are experimentally supported +// by this package in a limited capacity. Setting the EnableCompression option +// to true in Dialer or Upgrader will attempt to negotiate per message deflate +// support. +// +// var upgrader = websocket.Upgrader{ +// EnableCompression: true, +// } +// +// If compression was successfully negotiated with the connection's peer, any +// message received in compressed form will be automatically decompressed. +// All Read methods will return uncompressed bytes. +// +// Per message compression of messages written to a connection can be enabled +// or disabled by calling the corresponding Conn method: +// +// conn.EnableWriteCompression(false) +// +// Currently this package does not support compression with "context takeover". +// This means that messages must be compressed and decompressed in isolation, +// without retaining sliding window or dictionary state across messages. For +// more details refer to RFC 7692. +// +// Use of compression is experimental and may result in decreased performance. package websocket diff --git a/vendor/github.com/gorilla/websocket/example_test.go b/vendor/github.com/gorilla/websocket/example_test.go deleted file mode 100644 index ed22c4ae..00000000 --- a/vendor/github.com/gorilla/websocket/example_test.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket_test - -import ( - "log" - "net/http" - "testing" - - "github.com/gorilla/websocket" -) - -// The websocket.IsUnexpectedCloseError function is useful for identifying -// application and protocol errors. -// -// This server application works with a client application running in the -// browser. The client application does not explicitly close the websocket. The -// only expected close message from the client has the code -// websocket.CloseGoingAway. All other other close messages are likely the -// result of an application or protocol error and are logged to aid debugging. -func ExampleIsUnexpectedCloseError(err error, c *websocket.Conn, req *http.Request) { - for { - messageType, p, err := c.ReadMessage() - if err != nil { - if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) { - log.Printf("error: %v, user-agent: %v", err, req.Header.Get("User-Agent")) - } - return - } - processMesage(messageType, p) - } -} - -func processMesage(mt int, p []byte) {} - -// TestX prevents godoc from showing this entire file in the example. Remove -// this function when a second example is added. -func TestX(t *testing.T) {} diff --git a/vendor/github.com/gorilla/websocket/examples/autobahn/README.md b/vendor/github.com/gorilla/websocket/examples/autobahn/README.md deleted file mode 100644 index 075ac153..00000000 --- a/vendor/github.com/gorilla/websocket/examples/autobahn/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Test Server - -This package contains a server for the [Autobahn WebSockets Test Suite](http://autobahn.ws/testsuite). - -To test the server, run - - go run server.go - -and start the client test driver - - wstest -m fuzzingclient -s fuzzingclient.json - -When the client completes, it writes a report to reports/clients/index.html. diff --git a/vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json b/vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json deleted file mode 100644 index 27d5a5b1..00000000 --- a/vendor/github.com/gorilla/websocket/examples/autobahn/fuzzingclient.json +++ /dev/null @@ -1,14 +0,0 @@ - -{ - "options": {"failByDrop": false}, - "outdir": "./reports/clients", - "servers": [ - {"agent": "ReadAllWriteMessage", "url": "ws://localhost:9000/m", "options": {"version": 18}}, - {"agent": "ReadAllWrite", "url": "ws://localhost:9000/r", "options": {"version": 18}}, - {"agent": "CopyFull", "url": "ws://localhost:9000/f", "options": {"version": 18}}, - {"agent": "CopyWriterOnly", "url": "ws://localhost:9000/c", "options": {"version": 18}} - ], - "cases": ["*"], - "exclude-cases": [], - "exclude-agent-cases": {} -} diff --git a/vendor/github.com/gorilla/websocket/examples/autobahn/server.go b/vendor/github.com/gorilla/websocket/examples/autobahn/server.go deleted file mode 100644 index d96ac84d..00000000 --- a/vendor/github.com/gorilla/websocket/examples/autobahn/server.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Command server is a test server for the Autobahn WebSockets Test Suite. -package main - -import ( - "errors" - "flag" - "github.com/gorilla/websocket" - "io" - "log" - "net/http" - "time" - "unicode/utf8" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 4096, - WriteBufferSize: 4096, - CheckOrigin: func(r *http.Request) bool { - return true - }, -} - -// echoCopy echoes messages from the client using io.Copy. -func echoCopy(w http.ResponseWriter, r *http.Request, writerOnly bool) { - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Println("Upgrade:", err) - return - } - defer conn.Close() - for { - mt, r, err := conn.NextReader() - if err != nil { - if err != io.EOF { - log.Println("NextReader:", err) - } - return - } - if mt == websocket.TextMessage { - r = &validator{r: r} - } - w, err := conn.NextWriter(mt) - if err != nil { - log.Println("NextWriter:", err) - return - } - if mt == websocket.TextMessage { - r = &validator{r: r} - } - if writerOnly { - _, err = io.Copy(struct{ io.Writer }{w}, r) - } else { - _, err = io.Copy(w, r) - } - if err != nil { - if err == errInvalidUTF8 { - conn.WriteControl(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseInvalidFramePayloadData, ""), - time.Time{}) - } - log.Println("Copy:", err) - return - } - err = w.Close() - if err != nil { - log.Println("Close:", err) - return - } - } -} - -func echoCopyWriterOnly(w http.ResponseWriter, r *http.Request) { - echoCopy(w, r, true) -} - -func echoCopyFull(w http.ResponseWriter, r *http.Request) { - echoCopy(w, r, false) -} - -// echoReadAll echoes messages from the client by reading the entire message -// with ioutil.ReadAll. -func echoReadAll(w http.ResponseWriter, r *http.Request, writeMessage bool) { - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Println("Upgrade:", err) - return - } - defer conn.Close() - for { - mt, b, err := conn.ReadMessage() - if err != nil { - if err != io.EOF { - log.Println("NextReader:", err) - } - return - } - if mt == websocket.TextMessage { - if !utf8.Valid(b) { - conn.WriteControl(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseInvalidFramePayloadData, ""), - time.Time{}) - log.Println("ReadAll: invalid utf8") - } - } - if writeMessage { - err = conn.WriteMessage(mt, b) - if err != nil { - log.Println("WriteMessage:", err) - } - } else { - w, err := conn.NextWriter(mt) - if err != nil { - log.Println("NextWriter:", err) - return - } - if _, err := w.Write(b); err != nil { - log.Println("Writer:", err) - return - } - if err := w.Close(); err != nil { - log.Println("Close:", err) - return - } - } - } -} - -func echoReadAllWriter(w http.ResponseWriter, r *http.Request) { - echoReadAll(w, r, false) -} - -func echoReadAllWriteMessage(w http.ResponseWriter, r *http.Request) { - echoReadAll(w, r, true) -} - -func serveHome(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.Error(w, "Not found.", 404) - return - } - if r.Method != "GET" { - http.Error(w, "Method not allowed", 405) - return - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - io.WriteString(w, "Echo Server") -} - -var addr = flag.String("addr", ":9000", "http service address") - -func main() { - flag.Parse() - http.HandleFunc("/", serveHome) - http.HandleFunc("/c", echoCopyWriterOnly) - http.HandleFunc("/f", echoCopyFull) - http.HandleFunc("/r", echoReadAllWriter) - http.HandleFunc("/m", echoReadAllWriteMessage) - err := http.ListenAndServe(*addr, nil) - if err != nil { - log.Fatal("ListenAndServe: ", err) - } -} - -type validator struct { - state int - x rune - r io.Reader -} - -var errInvalidUTF8 = errors.New("invalid utf8") - -func (r *validator) Read(p []byte) (int, error) { - n, err := r.r.Read(p) - state := r.state - x := r.x - for _, b := range p[:n] { - state, x = decode(state, x, b) - if state == utf8Reject { - break - } - } - r.state = state - r.x = x - if state == utf8Reject || (err == io.EOF && state != utf8Accept) { - return n, errInvalidUTF8 - } - return n, err -} - -// UTF-8 decoder from http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ -// -// Copyright (c) 2008-2009 Bjoern Hoehrmann -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to -// deal in the Software without restriction, including without limitation the -// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -// sell copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -// IN THE SOFTWARE. -var utf8d = [...]byte{ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1f - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3f - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5f - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7f - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9f - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // a0..bf - 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // c0..df - 0xa, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // e0..ef - 0xb, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // f0..ff - 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 - 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 - 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 - 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // s7..s8 -} - -const ( - utf8Accept = 0 - utf8Reject = 1 -) - -func decode(state int, x rune, b byte) (int, rune) { - t := utf8d[b] - if state != utf8Accept { - x = rune(b&0x3f) | (x << 6) - } else { - x = rune((0xff >> t) & b) - } - state = int(utf8d[256+state*16+int(t)]) - return state, x -} diff --git a/vendor/github.com/gorilla/websocket/examples/chat/README.md b/vendor/github.com/gorilla/websocket/examples/chat/README.md deleted file mode 100644 index 5df3cf1a..00000000 --- a/vendor/github.com/gorilla/websocket/examples/chat/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Chat Example - -This application shows how to use use the -[websocket](https://github.com/gorilla/websocket) package and -[jQuery](http://jquery.com) to implement a simple web chat application. - -## Running the example - -The example requires a working Go development environment. The [Getting -Started](http://golang.org/doc/install) page describes how to install the -development environment. - -Once you have Go up and running, you can download, build and run the example -using the following commands. - - $ go get github.com/gorilla/websocket - $ cd `go list -f '{{.Dir}}' github.com/gorilla/websocket/examples/chat` - $ go run *.go - -To use the chat example, open http://localhost:8080/ in your browser. diff --git a/vendor/github.com/gorilla/websocket/examples/chat/conn.go b/vendor/github.com/gorilla/websocket/examples/chat/conn.go deleted file mode 100644 index 40fd38c2..00000000 --- a/vendor/github.com/gorilla/websocket/examples/chat/conn.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "github.com/gorilla/websocket" - "log" - "net/http" - "time" -) - -const ( - // Time allowed to write a message to the peer. - writeWait = 10 * time.Second - - // Time allowed to read the next pong message from the peer. - pongWait = 60 * time.Second - - // Send pings to peer with this period. Must be less than pongWait. - pingPeriod = (pongWait * 9) / 10 - - // Maximum message size allowed from peer. - maxMessageSize = 512 -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, -} - -// connection is an middleman between the websocket connection and the hub. -type connection struct { - // The websocket connection. - ws *websocket.Conn - - // Buffered channel of outbound messages. - send chan []byte -} - -// readPump pumps messages from the websocket connection to the hub. -func (c *connection) readPump() { - defer func() { - h.unregister <- c - c.ws.Close() - }() - c.ws.SetReadLimit(maxMessageSize) - c.ws.SetReadDeadline(time.Now().Add(pongWait)) - c.ws.SetPongHandler(func(string) error { c.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil }) - for { - _, message, err := c.ws.ReadMessage() - if err != nil { - if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) { - log.Printf("error: %v", err) - } - break - } - h.broadcast <- message - } -} - -// write writes a message with the given message type and payload. -func (c *connection) write(mt int, payload []byte) error { - c.ws.SetWriteDeadline(time.Now().Add(writeWait)) - return c.ws.WriteMessage(mt, payload) -} - -// writePump pumps messages from the hub to the websocket connection. -func (c *connection) writePump() { - ticker := time.NewTicker(pingPeriod) - defer func() { - ticker.Stop() - c.ws.Close() - }() - for { - select { - case message, ok := <-c.send: - if !ok { - c.write(websocket.CloseMessage, []byte{}) - return - } - if err := c.write(websocket.TextMessage, message); err != nil { - return - } - case <-ticker.C: - if err := c.write(websocket.PingMessage, []byte{}); err != nil { - return - } - } - } -} - -// serveWs handles websocket requests from the peer. -func serveWs(w http.ResponseWriter, r *http.Request) { - ws, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Println(err) - return - } - c := &connection{send: make(chan []byte, 256), ws: ws} - h.register <- c - go c.writePump() - c.readPump() -} diff --git a/vendor/github.com/gorilla/websocket/examples/chat/home.html b/vendor/github.com/gorilla/websocket/examples/chat/home.html deleted file mode 100644 index 29599225..00000000 --- a/vendor/github.com/gorilla/websocket/examples/chat/home.html +++ /dev/null @@ -1,92 +0,0 @@ - - - -Chat Example - - - - - -
-
- - -
- - diff --git a/vendor/github.com/gorilla/websocket/examples/chat/hub.go b/vendor/github.com/gorilla/websocket/examples/chat/hub.go deleted file mode 100644 index 449ba753..00000000 --- a/vendor/github.com/gorilla/websocket/examples/chat/hub.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -// hub maintains the set of active connections and broadcasts messages to the -// connections. -type hub struct { - // Registered connections. - connections map[*connection]bool - - // Inbound messages from the connections. - broadcast chan []byte - - // Register requests from the connections. - register chan *connection - - // Unregister requests from connections. - unregister chan *connection -} - -var h = hub{ - broadcast: make(chan []byte), - register: make(chan *connection), - unregister: make(chan *connection), - connections: make(map[*connection]bool), -} - -func (h *hub) run() { - for { - select { - case c := <-h.register: - h.connections[c] = true - case c := <-h.unregister: - if _, ok := h.connections[c]; ok { - delete(h.connections, c) - close(c.send) - } - case m := <-h.broadcast: - for c := range h.connections { - select { - case c.send <- m: - default: - close(c.send) - delete(h.connections, c) - } - } - } - } -} diff --git a/vendor/github.com/gorilla/websocket/examples/chat/main.go b/vendor/github.com/gorilla/websocket/examples/chat/main.go deleted file mode 100644 index 3c4448d7..00000000 --- a/vendor/github.com/gorilla/websocket/examples/chat/main.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "log" - "net/http" - "text/template" -) - -var addr = flag.String("addr", ":8080", "http service address") -var homeTempl = template.Must(template.ParseFiles("home.html")) - -func serveHome(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.Error(w, "Not found", 404) - return - } - if r.Method != "GET" { - http.Error(w, "Method not allowed", 405) - return - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - homeTempl.Execute(w, r.Host) -} - -func main() { - flag.Parse() - go h.run() - http.HandleFunc("/", serveHome) - http.HandleFunc("/ws", serveWs) - err := http.ListenAndServe(*addr, nil) - if err != nil { - log.Fatal("ListenAndServe: ", err) - } -} diff --git a/vendor/github.com/gorilla/websocket/examples/command/README.md b/vendor/github.com/gorilla/websocket/examples/command/README.md deleted file mode 100644 index c30d3979..00000000 --- a/vendor/github.com/gorilla/websocket/examples/command/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Command example - -This example connects a websocket connection to stdin and stdout of a command. -Received messages are written to stdin followed by a `\n`. Each line read from -from standard out is sent as a message to the client. - - $ go get github.com/gorilla/websocket - $ cd `go list -f '{{.Dir}}' github.com/gorilla/websocket/examples/command` - $ go run main.go - # Open http://localhost:8080/ . - -Try the following commands. - - # Echo sent messages to the output area. - $ go run main.go cat - - # Run a shell.Try sending "ls" and "cat main.go". - $ go run main.go sh - diff --git a/vendor/github.com/gorilla/websocket/examples/command/home.html b/vendor/github.com/gorilla/websocket/examples/command/home.html deleted file mode 100644 index 72fd02b2..00000000 --- a/vendor/github.com/gorilla/websocket/examples/command/home.html +++ /dev/null @@ -1,96 +0,0 @@ - - - -Command Example - - - - - -
-
- - -
- - diff --git a/vendor/github.com/gorilla/websocket/examples/command/main.go b/vendor/github.com/gorilla/websocket/examples/command/main.go deleted file mode 100644 index f3f022ed..00000000 --- a/vendor/github.com/gorilla/websocket/examples/command/main.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bufio" - "flag" - "io" - "log" - "net/http" - "os" - "os/exec" - "text/template" - "time" - - "github.com/gorilla/websocket" -) - -var ( - addr = flag.String("addr", "127.0.0.1:8080", "http service address") - cmdPath string - homeTempl = template.Must(template.ParseFiles("home.html")) -) - -const ( - // Time allowed to write a message to the peer. - writeWait = 10 * time.Second - - // Maximum message size allowed from peer. - maxMessageSize = 8192 - - // Time allowed to read the next pong message from the peer. - pongWait = 60 * time.Second - - // Send pings to peer with this period. Must be less than pongWait. - pingPeriod = (pongWait * 9) / 10 -) - -func pumpStdin(ws *websocket.Conn, w io.Writer) { - defer ws.Close() - ws.SetReadLimit(maxMessageSize) - ws.SetReadDeadline(time.Now().Add(pongWait)) - ws.SetPongHandler(func(string) error { ws.SetReadDeadline(time.Now().Add(pongWait)); return nil }) - for { - _, message, err := ws.ReadMessage() - if err != nil { - break - } - message = append(message, '\n') - if _, err := w.Write(message); err != nil { - break - } - } -} - -func pumpStdout(ws *websocket.Conn, r io.Reader, done chan struct{}) { - defer func() { - ws.Close() - close(done) - }() - s := bufio.NewScanner(r) - for s.Scan() { - ws.SetWriteDeadline(time.Now().Add(writeWait)) - if err := ws.WriteMessage(websocket.TextMessage, s.Bytes()); err != nil { - break - } - } - if s.Err() != nil { - log.Println("scan:", s.Err()) - } -} - -func ping(ws *websocket.Conn, done chan struct{}) { - ticker := time.NewTicker(pingPeriod) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)); err != nil { - log.Println("ping:", err) - } - case <-done: - return - } - } -} - -func internalError(ws *websocket.Conn, msg string, err error) { - log.Println(msg, err) - ws.WriteMessage(websocket.TextMessage, []byte("Internal server error.")) -} - -var upgrader = websocket.Upgrader{} - -func serveWs(w http.ResponseWriter, r *http.Request) { - ws, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Println("upgrade:", err) - return - } - - defer ws.Close() - - outr, outw, err := os.Pipe() - if err != nil { - internalError(ws, "stdout:", err) - return - } - defer outr.Close() - defer outw.Close() - - inr, inw, err := os.Pipe() - if err != nil { - internalError(ws, "stdin:", err) - return - } - defer inr.Close() - defer inw.Close() - - proc, err := os.StartProcess(cmdPath, flag.Args(), &os.ProcAttr{ - Files: []*os.File{inr, outw, outw}, - }) - if err != nil { - internalError(ws, "start:", err) - return - } - - inr.Close() - outw.Close() - - stdoutDone := make(chan struct{}) - go pumpStdout(ws, outr, stdoutDone) - go ping(ws, stdoutDone) - - pumpStdin(ws, inw) - - // Some commands will exit when stdin is closed. - inw.Close() - - // Other commands need a bonk on the head. - if err := proc.Signal(os.Interrupt); err != nil { - log.Println("inter:", err) - } - - select { - case <-stdoutDone: - case <-time.After(time.Second): - // A bigger bonk on the head. - if err := proc.Signal(os.Kill); err != nil { - log.Println("term:", err) - } - <-stdoutDone - } - - if _, err := proc.Wait(); err != nil { - log.Println("wait:", err) - } -} - -func serveHome(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.Error(w, "Not found", 404) - return - } - if r.Method != "GET" { - http.Error(w, "Method not allowed", 405) - return - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - homeTempl.Execute(w, r.Host) -} - -func main() { - flag.Parse() - if len(flag.Args()) < 1 { - log.Fatal("must specify at least one argument") - } - var err error - cmdPath, err = exec.LookPath(flag.Args()[0]) - if err != nil { - log.Fatal(err) - } - http.HandleFunc("/", serveHome) - http.HandleFunc("/ws", serveWs) - log.Fatal(http.ListenAndServe(*addr, nil)) -} diff --git a/vendor/github.com/gorilla/websocket/examples/echo/README.md b/vendor/github.com/gorilla/websocket/examples/echo/README.md deleted file mode 100644 index 6ad79ed7..00000000 --- a/vendor/github.com/gorilla/websocket/examples/echo/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Client and server example - -This example shows a simple client and server. - -The server echoes messages sent to it. The client sends a message every second -and prints all messages received. - -To run the example, start the server: - - $ go run server.go - -Next, start the client: - - $ go run client.go - -The server includes a simple web client. To use the client, open -http://127.0.0.1:8080 in the browser and follow the instructions on the page. diff --git a/vendor/github.com/gorilla/websocket/examples/echo/client.go b/vendor/github.com/gorilla/websocket/examples/echo/client.go deleted file mode 100644 index 6578094e..00000000 --- a/vendor/github.com/gorilla/websocket/examples/echo/client.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -import ( - "flag" - "log" - "net/url" - "os" - "os/signal" - "time" - - "github.com/gorilla/websocket" -) - -var addr = flag.String("addr", "localhost:8080", "http service address") - -func main() { - flag.Parse() - log.SetFlags(0) - - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, os.Interrupt) - - u := url.URL{Scheme: "ws", Host: *addr, Path: "/echo"} - log.Printf("connecting to %s", u.String()) - - c, _, err := websocket.DefaultDialer.Dial(u.String(), nil) - if err != nil { - log.Fatal("dial:", err) - } - defer c.Close() - - done := make(chan struct{}) - - go func() { - defer c.Close() - defer close(done) - for { - _, message, err := c.ReadMessage() - if err != nil { - log.Println("read:", err) - return - } - log.Printf("recv: %s", message) - } - }() - - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - - for { - select { - case t := <-ticker.C: - err := c.WriteMessage(websocket.TextMessage, []byte(t.String())) - if err != nil { - log.Println("write:", err) - return - } - case <-interrupt: - log.Println("interrupt") - // To cleanly close a connection, a client should send a close - // frame and wait for the server to close the connection. - err := c.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) - if err != nil { - log.Println("write close:", err) - return - } - select { - case <-done: - case <-time.After(time.Second): - } - c.Close() - return - } - } -} diff --git a/vendor/github.com/gorilla/websocket/examples/echo/server.go b/vendor/github.com/gorilla/websocket/examples/echo/server.go deleted file mode 100644 index a685b097..00000000 --- a/vendor/github.com/gorilla/websocket/examples/echo/server.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2015 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -import ( - "flag" - "html/template" - "log" - "net/http" - - "github.com/gorilla/websocket" -) - -var addr = flag.String("addr", "localhost:8080", "http service address") - -var upgrader = websocket.Upgrader{} // use default options - -func echo(w http.ResponseWriter, r *http.Request) { - c, err := upgrader.Upgrade(w, r, nil) - if err != nil { - log.Print("upgrade:", err) - return - } - defer c.Close() - for { - mt, message, err := c.ReadMessage() - if err != nil { - log.Println("read:", err) - break - } - log.Printf("recv: %s", message) - err = c.WriteMessage(mt, message) - if err != nil { - log.Println("write:", err) - break - } - } -} - -func home(w http.ResponseWriter, r *http.Request) { - homeTemplate.Execute(w, "ws://"+r.Host+"/echo") -} - -func main() { - flag.Parse() - log.SetFlags(0) - http.HandleFunc("/echo", echo) - http.HandleFunc("/", home) - log.Fatal(http.ListenAndServe(*addr, nil)) -} - -var homeTemplate = template.Must(template.New("").Parse(` - - - - - - - -
-

Click "Open" to create a connection to the server, -"Send" to send a message to the server and "Close" to close the connection. -You can change the message and send multiple times. -

-

- - -

- -

-		 -
-
- - -`)) diff --git a/vendor/github.com/gorilla/websocket/examples/filewatch/README.md b/vendor/github.com/gorilla/websocket/examples/filewatch/README.md deleted file mode 100644 index ca4931f3..00000000 --- a/vendor/github.com/gorilla/websocket/examples/filewatch/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# File Watch example. - -This example sends a file to the browser client for display whenever the file is modified. - - $ go get github.com/gorilla/websocket - $ cd `go list -f '{{.Dir}}' github.com/gorilla/websocket/examples/filewatch` - $ go run main.go - # Open http://localhost:8080/ . - # Modify the file to see it update in the browser. diff --git a/vendor/github.com/gorilla/websocket/examples/filewatch/main.go b/vendor/github.com/gorilla/websocket/examples/filewatch/main.go deleted file mode 100644 index a2c7b85f..00000000 --- a/vendor/github.com/gorilla/websocket/examples/filewatch/main.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "io/ioutil" - "log" - "net/http" - "os" - "strconv" - "text/template" - "time" - - "github.com/gorilla/websocket" -) - -const ( - // Time allowed to write the file to the client. - writeWait = 10 * time.Second - - // Time allowed to read the next pong message from the client. - pongWait = 60 * time.Second - - // Send pings to client with this period. Must be less than pongWait. - pingPeriod = (pongWait * 9) / 10 - - // Poll file for changes with this period. - filePeriod = 10 * time.Second -) - -var ( - addr = flag.String("addr", ":8080", "http service address") - homeTempl = template.Must(template.New("").Parse(homeHTML)) - filename string - upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - } -) - -func readFileIfModified(lastMod time.Time) ([]byte, time.Time, error) { - fi, err := os.Stat(filename) - if err != nil { - return nil, lastMod, err - } - if !fi.ModTime().After(lastMod) { - return nil, lastMod, nil - } - p, err := ioutil.ReadFile(filename) - if err != nil { - return nil, fi.ModTime(), err - } - return p, fi.ModTime(), nil -} - -func reader(ws *websocket.Conn) { - defer ws.Close() - ws.SetReadLimit(512) - ws.SetReadDeadline(time.Now().Add(pongWait)) - ws.SetPongHandler(func(string) error { ws.SetReadDeadline(time.Now().Add(pongWait)); return nil }) - for { - _, _, err := ws.ReadMessage() - if err != nil { - break - } - } -} - -func writer(ws *websocket.Conn, lastMod time.Time) { - lastError := "" - pingTicker := time.NewTicker(pingPeriod) - fileTicker := time.NewTicker(filePeriod) - defer func() { - pingTicker.Stop() - fileTicker.Stop() - ws.Close() - }() - for { - select { - case <-fileTicker.C: - var p []byte - var err error - - p, lastMod, err = readFileIfModified(lastMod) - - if err != nil { - if s := err.Error(); s != lastError { - lastError = s - p = []byte(lastError) - } - } else { - lastError = "" - } - - if p != nil { - ws.SetWriteDeadline(time.Now().Add(writeWait)) - if err := ws.WriteMessage(websocket.TextMessage, p); err != nil { - return - } - } - case <-pingTicker.C: - ws.SetWriteDeadline(time.Now().Add(writeWait)) - if err := ws.WriteMessage(websocket.PingMessage, []byte{}); err != nil { - return - } - } - } -} - -func serveWs(w http.ResponseWriter, r *http.Request) { - ws, err := upgrader.Upgrade(w, r, nil) - if err != nil { - if _, ok := err.(websocket.HandshakeError); !ok { - log.Println(err) - } - return - } - - var lastMod time.Time - if n, err := strconv.ParseInt(r.FormValue("lastMod"), 16, 64); err != nil { - lastMod = time.Unix(0, n) - } - - go writer(ws, lastMod) - reader(ws) -} - -func serveHome(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.Error(w, "Not found", 404) - return - } - if r.Method != "GET" { - http.Error(w, "Method not allowed", 405) - return - } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - p, lastMod, err := readFileIfModified(time.Time{}) - if err != nil { - p = []byte(err.Error()) - lastMod = time.Unix(0, 0) - } - var v = struct { - Host string - Data string - LastMod string - }{ - r.Host, - string(p), - strconv.FormatInt(lastMod.UnixNano(), 16), - } - homeTempl.Execute(w, &v) -} - -func main() { - flag.Parse() - if flag.NArg() != 1 { - log.Fatal("filename not specified") - } - filename = flag.Args()[0] - http.HandleFunc("/", serveHome) - http.HandleFunc("/ws", serveWs) - if err := http.ListenAndServe(*addr, nil); err != nil { - log.Fatal(err) - } -} - -const homeHTML = ` - - - WebSocket Example - - - 
{{.Data}}
- - - -` diff --git a/vendor/github.com/gorilla/websocket/json_test.go b/vendor/github.com/gorilla/websocket/json_test.go deleted file mode 100644 index 61100e48..00000000 --- a/vendor/github.com/gorilla/websocket/json_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket - -import ( - "bytes" - "encoding/json" - "io" - "reflect" - "testing" -) - -func TestJSON(t *testing.T) { - var buf bytes.Buffer - c := fakeNetConn{&buf, &buf} - wc := newConn(c, true, 1024, 1024) - rc := newConn(c, false, 1024, 1024) - - var actual, expect struct { - A int - B string - } - expect.A = 1 - expect.B = "hello" - - if err := wc.WriteJSON(&expect); err != nil { - t.Fatal("write", err) - } - - if err := rc.ReadJSON(&actual); err != nil { - t.Fatal("read", err) - } - - if !reflect.DeepEqual(&actual, &expect) { - t.Fatal("equal", actual, expect) - } -} - -func TestPartialJSONRead(t *testing.T) { - var buf bytes.Buffer - c := fakeNetConn{&buf, &buf} - wc := newConn(c, true, 1024, 1024) - rc := newConn(c, false, 1024, 1024) - - var v struct { - A int - B string - } - v.A = 1 - v.B = "hello" - - messageCount := 0 - - // Partial JSON values. - - data, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - for i := len(data) - 1; i >= 0; i-- { - if err := wc.WriteMessage(TextMessage, data[:i]); err != nil { - t.Fatal(err) - } - messageCount++ - } - - // Whitespace. - - if err := wc.WriteMessage(TextMessage, []byte(" ")); err != nil { - t.Fatal(err) - } - messageCount++ - - // Close. - - if err := wc.WriteMessage(CloseMessage, FormatCloseMessage(CloseNormalClosure, "")); err != nil { - t.Fatal(err) - } - - for i := 0; i < messageCount; i++ { - err := rc.ReadJSON(&v) - if err != io.ErrUnexpectedEOF { - t.Error("read", i, err) - } - } - - err = rc.ReadJSON(&v) - if _, ok := err.(*CloseError); !ok { - t.Error("final", err) - } -} - -func TestDeprecatedJSON(t *testing.T) { - var buf bytes.Buffer - c := fakeNetConn{&buf, &buf} - wc := newConn(c, true, 1024, 1024) - rc := newConn(c, false, 1024, 1024) - - var actual, expect struct { - A int - B string - } - expect.A = 1 - expect.B = "hello" - - if err := WriteJSON(wc, &expect); err != nil { - t.Fatal("write", err) - } - - if err := ReadJSON(rc, &actual); err != nil { - t.Fatal("read", err) - } - - if !reflect.DeepEqual(&actual, &expect) { - t.Fatal("equal", actual, expect) - } -} diff --git a/vendor/github.com/gorilla/websocket/mask.go b/vendor/github.com/gorilla/websocket/mask.go new file mode 100644 index 00000000..6a88bbc7 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/mask.go @@ -0,0 +1,55 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +// +build !appengine + +package websocket + +import "unsafe" + +const wordSize = int(unsafe.Sizeof(uintptr(0))) + +func maskBytes(key [4]byte, pos int, b []byte) int { + + // Mask one byte at a time for small buffers. + if len(b) < 2*wordSize { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 + } + + // Mask one byte at a time to word boundary. + if n := int(uintptr(unsafe.Pointer(&b[0]))) % wordSize; n != 0 { + n = wordSize - n + for i := range b[:n] { + b[i] ^= key[pos&3] + pos++ + } + b = b[n:] + } + + // Create aligned word size key. + var k [wordSize]byte + for i := range k { + k[i] = key[(pos+i)&3] + } + kw := *(*uintptr)(unsafe.Pointer(&k)) + + // Mask one word at a time. + n := (len(b) / wordSize) * wordSize + for i := 0; i < n; i += wordSize { + *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&b[0])) + uintptr(i))) ^= kw + } + + // Mask one byte at a time for remaining bytes. + b = b[n:] + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + + return pos & 3 +} diff --git a/vendor/github.com/gorilla/websocket/mask_safe.go b/vendor/github.com/gorilla/websocket/mask_safe.go new file mode 100644 index 00000000..2aac060e --- /dev/null +++ b/vendor/github.com/gorilla/websocket/mask_safe.go @@ -0,0 +1,15 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +// +build appengine + +package websocket + +func maskBytes(key [4]byte, pos int, b []byte) int { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 +} diff --git a/vendor/github.com/gorilla/websocket/prepared.go b/vendor/github.com/gorilla/websocket/prepared.go new file mode 100644 index 00000000..1efffbd1 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/prepared.go @@ -0,0 +1,103 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "net" + "sync" + "time" +) + +// PreparedMessage caches on the wire representations of a message payload. +// Use PreparedMessage to efficiently send a message payload to multiple +// connections. PreparedMessage is especially useful when compression is used +// because the CPU and memory expensive compression operation can be executed +// once for a given set of compression options. +type PreparedMessage struct { + messageType int + data []byte + err error + mu sync.Mutex + frames map[prepareKey]*preparedFrame +} + +// prepareKey defines a unique set of options to cache prepared frames in PreparedMessage. +type prepareKey struct { + isServer bool + compress bool + compressionLevel int +} + +// preparedFrame contains data in wire representation. +type preparedFrame struct { + once sync.Once + data []byte +} + +// NewPreparedMessage returns an initialized PreparedMessage. You can then send +// it to connection using WritePreparedMessage method. Valid wire +// representation will be calculated lazily only once for a set of current +// connection options. +func NewPreparedMessage(messageType int, data []byte) (*PreparedMessage, error) { + pm := &PreparedMessage{ + messageType: messageType, + frames: make(map[prepareKey]*preparedFrame), + data: data, + } + + // Prepare a plain server frame. + _, frameData, err := pm.frame(prepareKey{isServer: true, compress: false}) + if err != nil { + return nil, err + } + + // To protect against caller modifying the data argument, remember the data + // copied to the plain server frame. + pm.data = frameData[len(frameData)-len(data):] + return pm, nil +} + +func (pm *PreparedMessage) frame(key prepareKey) (int, []byte, error) { + pm.mu.Lock() + frame, ok := pm.frames[key] + if !ok { + frame = &preparedFrame{} + pm.frames[key] = frame + } + pm.mu.Unlock() + + var err error + frame.once.Do(func() { + // Prepare a frame using a 'fake' connection. + // TODO: Refactor code in conn.go to allow more direct construction of + // the frame. + mu := make(chan bool, 1) + mu <- true + var nc prepareConn + c := &Conn{ + conn: &nc, + mu: mu, + isServer: key.isServer, + compressionLevel: key.compressionLevel, + enableWriteCompression: true, + writeBuf: make([]byte, defaultWriteBufferSize+maxFrameHeaderSize), + } + if key.compress { + c.newCompressionWriter = compressNoContextTakeover + } + err = c.WriteMessage(pm.messageType, pm.data) + frame.data = nc.buf.Bytes() + }) + return pm.messageType, frame.data, err +} + +type prepareConn struct { + buf bytes.Buffer + net.Conn +} + +func (pc *prepareConn) Write(p []byte) (int, error) { return pc.buf.Write(p) } +func (pc *prepareConn) SetWriteDeadline(t time.Time) error { return nil } diff --git a/vendor/github.com/gorilla/websocket/server.go b/vendor/github.com/gorilla/websocket/server.go index 85616c79..3495e0f1 100644 --- a/vendor/github.com/gorilla/websocket/server.go +++ b/vendor/github.com/gorilla/websocket/server.go @@ -28,8 +28,9 @@ type Upgrader struct { HandshakeTimeout time.Duration // ReadBufferSize and WriteBufferSize specify I/O buffer sizes. If a buffer - // size is zero, then a default value of 4096 is used. The I/O buffer sizes - // do not limit the size of the messages that can be sent or received. + // size is zero, then buffers allocated by the HTTP server are used. The + // I/O buffer sizes do not limit the size of the messages that can be sent + // or received. ReadBufferSize, WriteBufferSize int // Subprotocols specifies the server's supported protocols in order of @@ -46,6 +47,12 @@ type Upgrader struct { // CheckOrigin is nil, the host in the Origin header must not be set or // must match the host of the request. CheckOrigin func(r *http.Request) bool + + // EnableCompression specify if the server should attempt to negotiate per + // message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool } func (u *Upgrader) returnError(w http.ResponseWriter, r *http.Request, status int, reason string) (*Conn, error) { @@ -53,6 +60,7 @@ func (u *Upgrader) returnError(w http.ResponseWriter, r *http.Request, status in if u.Error != nil { u.Error(w, r, status, err) } else { + w.Header().Set("Sec-Websocket-Version", "13") http.Error(w, http.StatusText(status), status) } return nil, err @@ -97,18 +105,23 @@ func (u *Upgrader) selectSubprotocol(r *http.Request, responseHeader http.Header // response. func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) { if r.Method != "GET" { - return u.returnError(w, r, http.StatusMethodNotAllowed, "websocket: method not GET") + return u.returnError(w, r, http.StatusMethodNotAllowed, "websocket: not a websocket handshake: request method is not GET") } - if values := r.Header["Sec-Websocket-Version"]; len(values) == 0 || values[0] != "13" { - return u.returnError(w, r, http.StatusBadRequest, "websocket: version != 13") + + if _, ok := responseHeader["Sec-Websocket-Extensions"]; ok { + return u.returnError(w, r, http.StatusInternalServerError, "websocket: application specific 'Sec-Websocket-Extensions' headers are unsupported") } if !tokenListContainsValue(r.Header, "Connection", "upgrade") { - return u.returnError(w, r, http.StatusBadRequest, "websocket: could not find connection header with token 'upgrade'") + return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: 'upgrade' token not found in 'Connection' header") } if !tokenListContainsValue(r.Header, "Upgrade", "websocket") { - return u.returnError(w, r, http.StatusBadRequest, "websocket: could not find upgrade header with token 'websocket'") + return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: 'websocket' token not found in 'Upgrade' header") + } + + if !tokenListContainsValue(r.Header, "Sec-Websocket-Version", "13") { + return u.returnError(w, r, http.StatusBadRequest, "websocket: unsupported version: 13 not found in 'Sec-Websocket-Version' header") } checkOrigin := u.CheckOrigin @@ -116,19 +129,30 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade checkOrigin = checkSameOrigin } if !checkOrigin(r) { - return u.returnError(w, r, http.StatusForbidden, "websocket: origin not allowed") + return u.returnError(w, r, http.StatusForbidden, "websocket: 'Origin' header value not allowed") } challengeKey := r.Header.Get("Sec-Websocket-Key") if challengeKey == "" { - return u.returnError(w, r, http.StatusBadRequest, "websocket: key missing or blank") + return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: `Sec-Websocket-Key' header is missing or blank") } subprotocol := u.selectSubprotocol(r, responseHeader) + // Negotiate PMCE + var compress bool + if u.EnableCompression { + for _, ext := range parseExtensions(r.Header) { + if ext[""] != "permessage-deflate" { + continue + } + compress = true + break + } + } + var ( netConn net.Conn - br *bufio.Reader err error ) @@ -136,21 +160,25 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade if !ok { return u.returnError(w, r, http.StatusInternalServerError, "websocket: response does not implement http.Hijacker") } - var rw *bufio.ReadWriter - netConn, rw, err = h.Hijack() + var brw *bufio.ReadWriter + netConn, brw, err = h.Hijack() if err != nil { return u.returnError(w, r, http.StatusInternalServerError, err.Error()) } - br = rw.Reader - if br.Buffered() > 0 { + if brw.Reader.Buffered() > 0 { netConn.Close() return nil, errors.New("websocket: client sent data before handshake is complete") } - c := newConn(netConn, true, u.ReadBufferSize, u.WriteBufferSize) + c := newConnBRW(netConn, true, u.ReadBufferSize, u.WriteBufferSize, brw) c.subprotocol = subprotocol + if compress { + c.newCompressionWriter = compressNoContextTakeover + c.newDecompressionReader = decompressNoContextTakeover + } + p := c.writeBuf[:0] p = append(p, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "...) p = append(p, computeAcceptKey(challengeKey)...) @@ -160,6 +188,9 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade p = append(p, c.subprotocol...) p = append(p, "\r\n"...) } + if compress { + p = append(p, "Sec-Websocket-Extensions: permessage-deflate; server_no_context_takeover; client_no_context_takeover\r\n"...) + } for k, vs := range responseHeader { if k == "Sec-Websocket-Protocol" { continue @@ -251,3 +282,10 @@ func Subprotocols(r *http.Request) []string { } return protocols } + +// IsWebSocketUpgrade returns true if the client requested upgrade to the +// WebSocket protocol. +func IsWebSocketUpgrade(r *http.Request) bool { + return tokenListContainsValue(r.Header, "Connection", "upgrade") && + tokenListContainsValue(r.Header, "Upgrade", "websocket") +} diff --git a/vendor/github.com/gorilla/websocket/server_test.go b/vendor/github.com/gorilla/websocket/server_test.go deleted file mode 100644 index ead0776a..00000000 --- a/vendor/github.com/gorilla/websocket/server_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket - -import ( - "net/http" - "reflect" - "testing" -) - -var subprotocolTests = []struct { - h string - protocols []string -}{ - {"", nil}, - {"foo", []string{"foo"}}, - {"foo,bar", []string{"foo", "bar"}}, - {"foo, bar", []string{"foo", "bar"}}, - {" foo, bar", []string{"foo", "bar"}}, - {" foo, bar ", []string{"foo", "bar"}}, -} - -func TestSubprotocols(t *testing.T) { - for _, st := range subprotocolTests { - r := http.Request{Header: http.Header{"Sec-Websocket-Protocol": {st.h}}} - protocols := Subprotocols(&r) - if !reflect.DeepEqual(st.protocols, protocols) { - t.Errorf("SubProtocols(%q) returned %#v, want %#v", st.h, protocols, st.protocols) - } - } -} diff --git a/vendor/github.com/gorilla/websocket/util.go b/vendor/github.com/gorilla/websocket/util.go index ffdc265e..9a4908df 100644 --- a/vendor/github.com/gorilla/websocket/util.go +++ b/vendor/github.com/gorilla/websocket/util.go @@ -13,19 +13,6 @@ import ( "strings" ) -// tokenListContainsValue returns true if the 1#token header with the given -// name contains token. -func tokenListContainsValue(header http.Header, name string, value string) bool { - for _, v := range header[name] { - for _, s := range strings.Split(v, ",") { - if strings.EqualFold(value, strings.TrimSpace(s)) { - return true - } - } - } - return false -} - var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") func computeAcceptKey(challengeKey string) string { @@ -42,3 +29,186 @@ func generateChallengeKey() (string, error) { } return base64.StdEncoding.EncodeToString(p), nil } + +// Octet types from RFC 2616. +var octetTypes [256]byte + +const ( + isTokenOctet = 1 << iota + isSpaceOctet +) + +func init() { + // From RFC 2616 + // + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t byte + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpaceOctet + } + if isChar && !isCtl && !isSeparator { + t |= isTokenOctet + } + octetTypes[c] = t + } +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpaceOctet == 0 { + break + } + } + return s[i:] +} + +func nextToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isTokenOctet == 0 { + break + } + } + return s[:i], s[i:] +} + +func nextTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return nextToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + 1; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j += 1 + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j += 1 + } + } + return "", "" + } + } + return "", "" +} + +// tokenListContainsValue returns true if the 1#token header with the given +// name contains token. +func tokenListContainsValue(header http.Header, name string, value string) bool { +headers: + for _, s := range header[name] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + s = skipSpace(s) + if s != "" && s[0] != ',' { + continue headers + } + if strings.EqualFold(t, value) { + return true + } + if s == "" { + continue headers + } + s = s[1:] + } + } + return false +} + +// parseExtensiosn parses WebSocket extensions from a header. +func parseExtensions(header http.Header) []map[string]string { + + // From RFC 6455: + // + // Sec-WebSocket-Extensions = extension-list + // extension-list = 1#extension + // extension = extension-token *( ";" extension-param ) + // extension-token = registered-token + // registered-token = token + // extension-param = token [ "=" (token | quoted-string) ] + // ;When using the quoted-string syntax variant, the value + // ;after quoted-string unescaping MUST conform to the + // ;'token' ABNF. + + var result []map[string]string +headers: + for _, s := range header["Sec-Websocket-Extensions"] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + ext := map[string]string{"": t} + for { + s = skipSpace(s) + if !strings.HasPrefix(s, ";") { + break + } + var k string + k, s = nextToken(skipSpace(s[1:])) + if k == "" { + continue headers + } + s = skipSpace(s) + var v string + if strings.HasPrefix(s, "=") { + v, s = nextTokenOrQuoted(skipSpace(s[1:])) + s = skipSpace(s) + } + if s != "" && s[0] != ',' && s[0] != ';' { + continue headers + } + ext[k] = v + } + if s != "" && s[0] != ',' { + continue headers + } + result = append(result, ext) + if s == "" { + continue headers + } + s = s[1:] + } + } + return result +} diff --git a/vendor/github.com/gorilla/websocket/util_test.go b/vendor/github.com/gorilla/websocket/util_test.go deleted file mode 100644 index 91f70ceb..00000000 --- a/vendor/github.com/gorilla/websocket/util_test.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2014 The Gorilla WebSocket Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package websocket - -import ( - "net/http" - "testing" -) - -var tokenListContainsValueTests = []struct { - value string - ok bool -}{ - {"WebSocket", true}, - {"WEBSOCKET", true}, - {"websocket", true}, - {"websockets", false}, - {"x websocket", false}, - {"websocket x", false}, - {"other,websocket,more", true}, - {"other, websocket, more", true}, -} - -func TestTokenListContainsValue(t *testing.T) { - for _, tt := range tokenListContainsValueTests { - h := http.Header{"Upgrade": {tt.value}} - ok := tokenListContainsValue(h, "Upgrade", "websocket") - if ok != tt.ok { - t.Errorf("tokenListContainsValue(h, n, %q) = %v, want %v", tt.value, ok, tt.ok) - } - } -} diff --git a/vendor/github.com/hashicorp/hcl/Makefile b/vendor/github.com/hashicorp/hcl/Makefile index ad404a81..84fd743f 100644 --- a/vendor/github.com/hashicorp/hcl/Makefile +++ b/vendor/github.com/hashicorp/hcl/Makefile @@ -6,6 +6,7 @@ fmt: generate go fmt ./... test: generate + go get -t ./... go test $(TEST) $(TESTARGS) generate: diff --git a/vendor/github.com/hashicorp/hcl/README.md b/vendor/github.com/hashicorp/hcl/README.md index acec6662..c8223326 100644 --- a/vendor/github.com/hashicorp/hcl/README.md +++ b/vendor/github.com/hashicorp/hcl/README.md @@ -29,7 +29,7 @@ and some people wanted machine-friendly languages. JSON fits a nice balance in this, but is fairly verbose and most importantly doesn't support comments. With YAML, we found that beginners had a really hard time determining what the actual structure was, and -ended up guessing more than not whether to use a hyphen, colon, etc. +ended up guessing more often than not whether to use a hyphen, colon, etc. in order to represent some configuration key. Full programming languages such as Ruby enable complex behavior @@ -81,9 +81,20 @@ FOO * Boolean values: `true`, `false` * Arrays can be made by wrapping it in `[]`. Example: - `["foo", "bar", 42]`. Arrays can contain primitives - and other arrays, but cannot contain objects. Objects must - use the block syntax shown below. + `["foo", "bar", 42]`. Arrays can contain primitives, + other arrays, and objects. As an alternative, lists + of objects can be created with repeated blocks, using + this structure: + + ```hcl + service { + key = "value" + } + + service { + key = "value" + } + ``` Objects and nested objects are created using the structure shown below: @@ -92,6 +103,16 @@ variable "ami" { description = "the AMI to use" } ``` +This would be equivalent to the following json: +``` json +{ + "variable": { + "ami": { + "description": "the AMI to use" + } + } +} +``` ## Thanks diff --git a/vendor/github.com/hashicorp/hcl/appveyor.yml b/vendor/github.com/hashicorp/hcl/appveyor.yml new file mode 100644 index 00000000..3c8cdf8e --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/appveyor.yml @@ -0,0 +1,19 @@ +version: "build-{branch}-{build}" +image: Visual Studio 2015 +clone_folder: c:\gopath\src\github.com\hashicorp\hcl +environment: + GOPATH: c:\gopath +init: + - git config --global core.autocrlf true +install: +- cmd: >- + echo %Path% + + go version + + go env + + go get -t ./... + +build_script: +- cmd: go test -v ./... diff --git a/vendor/github.com/hashicorp/hcl/decoder.go b/vendor/github.com/hashicorp/hcl/decoder.go index b0c3fc6d..0b39c1b9 100644 --- a/vendor/github.com/hashicorp/hcl/decoder.go +++ b/vendor/github.com/hashicorp/hcl/decoder.go @@ -21,6 +21,17 @@ var ( nodeType reflect.Type = findNodeType() ) +// Unmarshal accepts a byte slice as input and writes the +// data to the value pointed to by v. +func Unmarshal(bs []byte, v interface{}) error { + root, err := parse(bs) + if err != nil { + return err + } + + return DecodeObject(v, root) +} + // Decode reads the given input and decodes it into the structure // given by `out`. func Decode(out interface{}, in string) error { @@ -80,7 +91,7 @@ func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error return d.decodeBool(name, node, result) case reflect.Float64: return d.decodeFloat(name, node, result) - case reflect.Int: + case reflect.Int, reflect.Int32, reflect.Int64: return d.decodeInt(name, node, result) case reflect.Interface: // When we see an interface, we make our own thing @@ -153,7 +164,11 @@ func (d *decoder) decodeInt(name string, node ast.Node, result reflect.Value) er return err } - result.Set(reflect.ValueOf(int(v))) + if result.Kind() == reflect.Interface { + result.Set(reflect.ValueOf(int(v))) + } else { + result.SetInt(v) + } return nil case token.STRING: v, err := strconv.ParseInt(n.Token.Value().(string), 0, 0) @@ -161,7 +176,11 @@ func (d *decoder) decodeInt(name string, node ast.Node, result reflect.Value) er return err } - result.Set(reflect.ValueOf(int(v))) + if result.Kind() == reflect.Interface { + result.Set(reflect.ValueOf(int(v))) + } else { + result.SetInt(v) + } return nil } } @@ -326,6 +345,14 @@ func (d *decoder) decodeMap(name string, node ast.Node, result reflect.Value) er continue } + // github.com/hashicorp/terraform/issue/5740 + if len(item.Keys) == 0 { + return &parser.PosError{ + Pos: node.Pos(), + Err: fmt.Errorf("%s: map must have string keys", name), + } + } + // Get the key we're dealing with, which is the first item keyStr := item.Keys[0].Token.Value().(string) @@ -390,7 +417,6 @@ func (d *decoder) decodeSlice(name string, node ast.Node, result reflect.Value) if result.Kind() == reflect.Interface { result = result.Elem() } - // Create the slice if it isn't nil resultType := result.Type() resultElemType := resultType.Elem() @@ -424,6 +450,12 @@ func (d *decoder) decodeSlice(name string, node ast.Node, result reflect.Value) // Decode val := reflect.Indirect(reflect.New(resultElemType)) + + // if item is an object that was decoded from ambiguous JSON and + // flattened, make sure it's expanded if it needs to decode into a + // defined structure. + item := expandObject(item, val) + if err := d.decode(fieldName, item, val); err != nil { return err } @@ -436,6 +468,57 @@ func (d *decoder) decodeSlice(name string, node ast.Node, result reflect.Value) return nil } +// expandObject detects if an ambiguous JSON object was flattened to a List which +// should be decoded into a struct, and expands the ast to properly deocode. +func expandObject(node ast.Node, result reflect.Value) ast.Node { + item, ok := node.(*ast.ObjectItem) + if !ok { + return node + } + + elemType := result.Type() + + // our target type must be a struct + switch elemType.Kind() { + case reflect.Ptr: + switch elemType.Elem().Kind() { + case reflect.Struct: + //OK + default: + return node + } + case reflect.Struct: + //OK + default: + return node + } + + // A list value will have a key and field name. If it had more fields, + // it wouldn't have been flattened. + if len(item.Keys) != 2 { + return node + } + + keyToken := item.Keys[0].Token + item.Keys = item.Keys[1:] + + // we need to un-flatten the ast enough to decode + newNode := &ast.ObjectItem{ + Keys: []*ast.ObjectKey{ + &ast.ObjectKey{ + Token: keyToken, + }, + }, + Val: &ast.ObjectType{ + List: &ast.ObjectList{ + Items: []*ast.ObjectItem{item}, + }, + }, + } + + return newNode +} + func (d *decoder) decodeString(name string, node ast.Node, result reflect.Value) error { switch n := node.(type) { case *ast.LiteralType: @@ -466,6 +549,14 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) node = ot.List } + // Handle the special case where the object itself is a literal. Previously + // the yacc parser would always ensure top-level elements were arrays. The new + // parser does not make the same guarantees, thus we need to convert any + // top-level literal elements into a list. + if _, ok := node.(*ast.LiteralType); ok && item != nil { + node = &ast.ObjectList{Items: []*ast.ObjectItem{item}} + } + list, ok := node.(*ast.ObjectList) if !ok { return &parser.PosError{ @@ -490,6 +581,12 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) structType := structVal.Type() for i := 0; i < structType.NumField(); i++ { fieldType := structType.Field(i) + tagParts := strings.Split(fieldType.Tag.Get(tagName), ",") + + // Ignore fields with tag name "-" + if tagParts[0] == "-" { + continue + } if fieldType.Anonymous { fieldKind := fieldType.Type.Kind() @@ -504,7 +601,6 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) // We have an embedded field. We "squash" the fields down // if specified in the tag. squash := false - tagParts := strings.Split(fieldType.Tag.Get(tagName), ",") for _, tag := range tagParts[1:] { if tag == "squash" { squash = true @@ -574,6 +670,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) // match (only object with the field), then we decode it exactly. // If it is a prefix match, then we decode the matches. filter := list.Filter(fieldName) + prefixMatches := filter.Children() matches := filter.Elem() if len(matches.Items) == 0 && len(prefixMatches.Items) == 0 { diff --git a/vendor/github.com/hashicorp/hcl/decoder_test.go b/vendor/github.com/hashicorp/hcl/decoder_test.go deleted file mode 100644 index e976271d..00000000 --- a/vendor/github.com/hashicorp/hcl/decoder_test.go +++ /dev/null @@ -1,669 +0,0 @@ -package hcl - -import ( - "io/ioutil" - "path/filepath" - "reflect" - "testing" - - "github.com/hashicorp/hcl/hcl/ast" -) - -func TestDecode_interface(t *testing.T) { - cases := []struct { - File string - Err bool - Out interface{} - }{ - { - "basic.hcl", - false, - map[string]interface{}{ - "foo": "bar", - "bar": "${file(\"bing/bong.txt\")}", - }, - }, - { - "basic_squish.hcl", - false, - map[string]interface{}{ - "foo": "bar", - "bar": "${file(\"bing/bong.txt\")}", - "foo-bar": "baz", - }, - }, - { - "empty.hcl", - false, - map[string]interface{}{ - "resource": []map[string]interface{}{ - map[string]interface{}{ - "foo": []map[string]interface{}{ - map[string]interface{}{}, - }, - }, - }, - }, - }, - { - "tfvars.hcl", - false, - map[string]interface{}{ - "regularvar": "Should work", - "map.key1": "Value", - "map.key2": "Other value", - }, - }, - { - "escape.hcl", - false, - map[string]interface{}{ - "foo": "bar\"baz\\n", - }, - }, - { - "interpolate_escape.hcl", - false, - map[string]interface{}{ - "foo": "${file(\"bing/bong.txt\")}", - }, - }, - { - "float.hcl", - false, - map[string]interface{}{ - "a": 1.02, - }, - }, - { - "multiline_bad.hcl", - true, - nil, - }, - { - "multiline_no_marker.hcl", - true, - nil, - }, - { - "multiline.hcl", - false, - map[string]interface{}{"foo": "bar\nbaz\n"}, - }, - { - "multiline_no_eof.hcl", - false, - map[string]interface{}{"foo": "bar\nbaz\n", "key": "value"}, - }, - { - "multiline.json", - false, - map[string]interface{}{"foo": "bar\nbaz"}, - }, - { - "scientific.json", - false, - map[string]interface{}{ - "a": 1e-10, - "b": 1e+10, - "c": 1e10, - "d": 1.2e-10, - "e": 1.2e+10, - "f": 1.2e10, - }, - }, - { - "scientific.hcl", - false, - map[string]interface{}{ - "a": 1e-10, - "b": 1e+10, - "c": 1e10, - "d": 1.2e-10, - "e": 1.2e+10, - "f": 1.2e10, - }, - }, - { - "terraform_heroku.hcl", - false, - map[string]interface{}{ - "name": "terraform-test-app", - "config_vars": []map[string]interface{}{ - map[string]interface{}{ - "FOO": "bar", - }, - }, - }, - }, - { - "structure_multi.hcl", - false, - map[string]interface{}{ - "foo": []map[string]interface{}{ - map[string]interface{}{ - "baz": []map[string]interface{}{ - map[string]interface{}{"key": 7}, - }, - }, - map[string]interface{}{ - "bar": []map[string]interface{}{ - map[string]interface{}{"key": 12}, - }, - }, - }, - }, - }, - { - "structure_multi.json", - false, - map[string]interface{}{ - "foo": []map[string]interface{}{ - map[string]interface{}{ - "baz": []map[string]interface{}{ - map[string]interface{}{"key": 7}, - }, - }, - map[string]interface{}{ - "bar": []map[string]interface{}{ - map[string]interface{}{"key": 12}, - }, - }, - }, - }, - }, - { - "structure_list.hcl", - false, - map[string]interface{}{ - "foo": []map[string]interface{}{ - map[string]interface{}{ - "key": 7, - }, - map[string]interface{}{ - "key": 12, - }, - }, - }, - }, - { - "structure_list.json", - false, - map[string]interface{}{ - "foo": []map[string]interface{}{ - map[string]interface{}{ - "key": 7, - }, - map[string]interface{}{ - "key": 12, - }, - }, - }, - }, - { - "structure_list_deep.json", - false, - map[string]interface{}{ - "bar": []map[string]interface{}{ - map[string]interface{}{ - "foo": []map[string]interface{}{ - map[string]interface{}{ - "name": "terraform_example", - "ingress": []map[string]interface{}{ - map[string]interface{}{ - "from_port": 22, - }, - map[string]interface{}{ - "from_port": 80, - }, - }, - }, - }, - }, - }, - }, - }, - - { - "nested_block_comment.hcl", - false, - map[string]interface{}{ - "bar": "value", - }, - }, - - { - "unterminated_block_comment.hcl", - true, - nil, - }, - - { - "unterminated_brace.hcl", - true, - nil, - }, - - { - "object_list.json", - false, - map[string]interface{}{ - "resource": []map[string]interface{}{ - map[string]interface{}{ - "aws_instance": []map[string]interface{}{ - map[string]interface{}{ - "db": []map[string]interface{}{ - map[string]interface{}{ - "vpc": "foo", - "provisioner": []map[string]interface{}{ - map[string]interface{}{ - "file": []map[string]interface{}{ - map[string]interface{}{ - "source": "foo", - "destination": "bar", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - for _, tc := range cases { - t.Logf("Testing: %s", tc.File) - d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.File)) - if err != nil { - t.Fatalf("err: %s", err) - } - - var out interface{} - err = Decode(&out, string(d)) - if (err != nil) != tc.Err { - t.Fatalf("Input: %s\n\nError: %s", tc.File, err) - } - - if !reflect.DeepEqual(out, tc.Out) { - t.Fatalf("Input: %s. Actual, Expected.\n\n%#v\n\n%#v", tc.File, out, tc.Out) - } - } -} - -func TestDecode_equal(t *testing.T) { - cases := []struct { - One, Two string - }{ - { - "basic.hcl", - "basic.json", - }, - { - "float.hcl", - "float.json", - }, - /* - { - "structure.hcl", - "structure.json", - }, - */ - { - "structure.hcl", - "structure_flat.json", - }, - { - "terraform_heroku.hcl", - "terraform_heroku.json", - }, - } - - for _, tc := range cases { - p1 := filepath.Join(fixtureDir, tc.One) - p2 := filepath.Join(fixtureDir, tc.Two) - - d1, err := ioutil.ReadFile(p1) - if err != nil { - t.Fatalf("err: %s", err) - } - - d2, err := ioutil.ReadFile(p2) - if err != nil { - t.Fatalf("err: %s", err) - } - - var i1, i2 interface{} - err = Decode(&i1, string(d1)) - if err != nil { - t.Fatalf("err: %s", err) - } - - err = Decode(&i2, string(d2)) - if err != nil { - t.Fatalf("err: %s", err) - } - - if !reflect.DeepEqual(i1, i2) { - t.Fatalf( - "%s != %s\n\n%#v\n\n%#v", - tc.One, tc.Two, - i1, i2) - } - } -} - -func TestDecode_flatMap(t *testing.T) { - var val map[string]map[string]string - - err := Decode(&val, testReadFile(t, "structure_flatmap.hcl")) - if err != nil { - t.Fatalf("err: %s", err) - } - - expected := map[string]map[string]string{ - "foo": map[string]string{ - "foo": "bar", - "key": "7", - }, - } - - if !reflect.DeepEqual(val, expected) { - t.Fatalf("Actual: %#v\n\nExpected: %#v", val, expected) - } -} - -func TestDecode_structure(t *testing.T) { - type V struct { - Key int - Foo string - } - - var actual V - - err := Decode(&actual, testReadFile(t, "flat.hcl")) - if err != nil { - t.Fatalf("err: %s", err) - } - - expected := V{ - Key: 7, - Foo: "bar", - } - - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("Actual: %#v\n\nExpected: %#v", actual, expected) - } -} - -func TestDecode_structurePtr(t *testing.T) { - type V struct { - Key int - Foo string - } - - var actual *V - - err := Decode(&actual, testReadFile(t, "flat.hcl")) - if err != nil { - t.Fatalf("err: %s", err) - } - - expected := &V{ - Key: 7, - Foo: "bar", - } - - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("Actual: %#v\n\nExpected: %#v", actual, expected) - } -} - -func TestDecode_structureArray(t *testing.T) { - // This test is extracted from a failure in Consul (consul.io), - // hence the interesting structure naming. - - type KeyPolicyType string - - type KeyPolicy struct { - Prefix string `hcl:",key"` - Policy KeyPolicyType - } - - type Policy struct { - Keys []KeyPolicy `hcl:"key,expand"` - } - - expected := Policy{ - Keys: []KeyPolicy{ - KeyPolicy{ - Prefix: "", - Policy: "read", - }, - KeyPolicy{ - Prefix: "foo/", - Policy: "write", - }, - KeyPolicy{ - Prefix: "foo/bar/", - Policy: "read", - }, - KeyPolicy{ - Prefix: "foo/bar/baz", - Policy: "deny", - }, - }, - } - - files := []string{ - "decode_policy.hcl", - "decode_policy.json", - } - - for _, f := range files { - var actual Policy - - err := Decode(&actual, testReadFile(t, f)) - if err != nil { - t.Fatalf("Input: %s\n\nerr: %s", f, err) - } - - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("Input: %s\n\nActual: %#v\n\nExpected: %#v", f, actual, expected) - } - } -} - -func TestDecode_sliceExpand(t *testing.T) { - type testInner struct { - Name string `hcl:",key"` - Key string - } - - type testStruct struct { - Services []testInner `hcl:"service,expand"` - } - - expected := testStruct{ - Services: []testInner{ - testInner{ - Name: "my-service-0", - Key: "value", - }, - testInner{ - Name: "my-service-1", - Key: "value", - }, - }, - } - - files := []string{ - "slice_expand.hcl", - } - - for _, f := range files { - t.Logf("Testing: %s", f) - - var actual testStruct - err := Decode(&actual, testReadFile(t, f)) - if err != nil { - t.Fatalf("Input: %s\n\nerr: %s", f, err) - } - - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("Input: %s\n\nActual: %#v\n\nExpected: %#v", f, actual, expected) - } - } -} - -func TestDecode_structureMap(t *testing.T) { - // This test is extracted from a failure in Terraform (terraform.io), - // hence the interesting structure naming. - - type hclVariable struct { - Default interface{} - Description string - Fields []string `hcl:",decodedFields"` - } - - type rawConfig struct { - Variable map[string]hclVariable - } - - expected := rawConfig{ - Variable: map[string]hclVariable{ - "foo": hclVariable{ - Default: "bar", - Description: "bar", - Fields: []string{"Default", "Description"}, - }, - - "amis": hclVariable{ - Default: []map[string]interface{}{ - map[string]interface{}{ - "east": "foo", - }, - }, - Fields: []string{"Default"}, - }, - }, - } - - files := []string{ - "decode_tf_variable.hcl", - "decode_tf_variable.json", - } - - for _, f := range files { - t.Logf("Testing: %s", f) - - var actual rawConfig - err := Decode(&actual, testReadFile(t, f)) - if err != nil { - t.Fatalf("Input: %s\n\nerr: %s", f, err) - } - - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("Input: %s\n\nActual: %#v\n\nExpected: %#v", f, actual, expected) - } - } -} - -func TestDecode_interfaceNonPointer(t *testing.T) { - var value interface{} - err := Decode(value, testReadFile(t, "basic_int_string.hcl")) - if err == nil { - t.Fatal("should error") - } -} - -func TestDecode_intString(t *testing.T) { - var value struct { - Count int - } - - err := Decode(&value, testReadFile(t, "basic_int_string.hcl")) - if err != nil { - t.Fatalf("err: %s", err) - } - - if value.Count != 3 { - t.Fatalf("bad: %#v", value.Count) - } -} - -func TestDecode_Node(t *testing.T) { - // given - var value struct { - Content ast.Node - Nested struct { - Content ast.Node - } - } - - content := ` -content { - hello = "world" -} -` - - // when - err := Decode(&value, content) - - // then - if err != nil { - t.Errorf("unable to decode content, %v", err) - return - } - - // verify ast.Node can be decoded later - var v map[string]interface{} - err = DecodeObject(&v, value.Content) - if err != nil { - t.Errorf("unable to decode content, %v", err) - return - } - - if v["hello"] != "world" { - t.Errorf("expected mapping to be returned") - } -} - -func TestDecode_NestedNode(t *testing.T) { - // given - var value struct { - Nested struct { - Content ast.Node - } - } - - content := ` -nested "content" { - hello = "world" -} -` - - // when - err := Decode(&value, content) - - // then - if err != nil { - t.Errorf("unable to decode content, %v", err) - return - } - - // verify ast.Node can be decoded later - var v map[string]interface{} - err = DecodeObject(&v, value.Nested.Content) - if err != nil { - t.Errorf("unable to decode content, %v", err) - return - } - - if v["hello"] != "world" { - t.Errorf("expected mapping to be returned") - } -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/ast/ast.go b/vendor/github.com/hashicorp/hcl/hcl/ast/ast.go index f8bb71a0..6e5ef654 100644 --- a/vendor/github.com/hashicorp/hcl/hcl/ast/ast.go +++ b/vendor/github.com/hashicorp/hcl/hcl/ast/ast.go @@ -133,6 +133,12 @@ type ObjectItem struct { } func (o *ObjectItem) Pos() token.Pos { + // I'm not entirely sure what causes this, but removing this causes + // a test failure. We should investigate at some point. + if len(o.Keys) == 0 { + return token.Pos{} + } + return o.Keys[0].Pos() } @@ -150,7 +156,8 @@ func (o *ObjectKey) Pos() token.Pos { type LiteralType struct { Token token.Token - // associated line comment, only when used in a list + // comment types, only used when in a list + LeadComment *CommentGroup LineComment *CommentGroup } @@ -208,4 +215,5 @@ func (c *CommentGroup) Pos() token.Pos { // GoStringer //------------------------------------------------------------------- -func (o *ObjectKey) GoString() string { return fmt.Sprintf("*%#v", *o) } +func (o *ObjectKey) GoString() string { return fmt.Sprintf("*%#v", *o) } +func (o *ObjectList) GoString() string { return fmt.Sprintf("*%#v", *o) } diff --git a/vendor/github.com/hashicorp/hcl/hcl/ast/ast_test.go b/vendor/github.com/hashicorp/hcl/hcl/ast/ast_test.go deleted file mode 100644 index 942256ca..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/ast/ast_test.go +++ /dev/null @@ -1,200 +0,0 @@ -package ast - -import ( - "reflect" - "strings" - "testing" - - "github.com/hashicorp/hcl/hcl/token" -) - -func TestObjectListFilter(t *testing.T) { - var cases = []struct { - Filter []string - Input []*ObjectItem - Output []*ObjectItem - }{ - { - []string{"foo"}, - []*ObjectItem{ - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{ - Token: token.Token{Type: token.STRING, Text: `"foo"`}, - }, - }, - }, - }, - []*ObjectItem{ - &ObjectItem{ - Keys: []*ObjectKey{}, - }, - }, - }, - - { - []string{"foo"}, - []*ObjectItem{ - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"foo"`}}, - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"bar"`}}, - }, - }, - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"baz"`}}, - }, - }, - }, - []*ObjectItem{ - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"bar"`}}, - }, - }, - }, - }, - } - - for _, tc := range cases { - input := &ObjectList{Items: tc.Input} - expected := &ObjectList{Items: tc.Output} - if actual := input.Filter(tc.Filter...); !reflect.DeepEqual(actual, expected) { - t.Fatalf("in order: input, expected, actual\n\n%#v\n\n%#v\n\n%#v", input, expected, actual) - } - } -} - -func TestWalk(t *testing.T) { - items := []*ObjectItem{ - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"foo"`}}, - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"bar"`}}, - }, - Val: &LiteralType{Token: token.Token{Type: token.STRING, Text: `"example"`}}, - }, - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"baz"`}}, - }, - }, - } - - node := &ObjectList{Items: items} - - order := []string{ - "*ast.ObjectList", - "*ast.ObjectItem", - "*ast.ObjectKey", - "*ast.ObjectKey", - "*ast.LiteralType", - "*ast.ObjectItem", - "*ast.ObjectKey", - } - count := 0 - - Walk(node, func(n Node) (Node, bool) { - if n == nil { - return n, false - } - - typeName := reflect.TypeOf(n).String() - if order[count] != typeName { - t.Errorf("expected '%s' got: '%s'", order[count], typeName) - } - count++ - return n, true - }) -} - -func TestWalkEquality(t *testing.T) { - items := []*ObjectItem{ - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"foo"`}}, - }, - }, - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"bar"`}}, - }, - }, - } - - node := &ObjectList{Items: items} - - rewritten := Walk(node, func(n Node) (Node, bool) { return n, true }) - - newNode, ok := rewritten.(*ObjectList) - if !ok { - t.Fatalf("expected Objectlist, got %T", rewritten) - } - - if !reflect.DeepEqual(node, newNode) { - t.Fatal("rewritten node is not equal to the given node") - } - - if len(newNode.Items) != 2 { - t.Error("expected newNode length 2, got: %d", len(newNode.Items)) - } - - expected := []string{ - `"foo"`, - `"bar"`, - } - - for i, item := range newNode.Items { - if len(item.Keys) != 1 { - t.Error("expected keys newNode length 1, got: %d", len(item.Keys)) - } - - if item.Keys[0].Token.Text != expected[i] { - t.Errorf("expected key %s, got %s", expected[i], item.Keys[0].Token.Text) - } - - if item.Val != nil { - t.Errorf("expected item value should be nil") - } - } -} - -func TestWalkRewrite(t *testing.T) { - items := []*ObjectItem{ - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"foo"`}}, - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"bar"`}}, - }, - }, - &ObjectItem{ - Keys: []*ObjectKey{ - &ObjectKey{Token: token.Token{Type: token.STRING, Text: `"baz"`}}, - }, - }, - } - - node := &ObjectList{Items: items} - - suffix := "_example" - node = Walk(node, func(n Node) (Node, bool) { - switch i := n.(type) { - case *ObjectKey: - i.Token.Text = i.Token.Text + suffix - n = i - } - return n, true - }).(*ObjectList) - - Walk(node, func(n Node) (Node, bool) { - switch i := n.(type) { - case *ObjectKey: - if !strings.HasSuffix(i.Token.Text, suffix) { - t.Errorf("Token '%s' should have suffix: %s", i.Token.Text, suffix) - } - } - return n, true - }) - -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd.go b/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd.go deleted file mode 100644 index afc1e4eb..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd.go +++ /dev/null @@ -1,164 +0,0 @@ -// Derivative work from: -// - https://golang.org/src/cmd/gofmt/gofmt.go -// - https://github.com/fatih/hclfmt - -package fmtcmd - -import ( - "bytes" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/hashicorp/hcl/hcl/printer" -) - -var ( - ErrWriteStdin = errors.New("cannot use write option with standard input") -) - -type Options struct { - List bool // list files whose formatting differs - Write bool // write result to (source) file instead of stdout - Diff bool // display diffs instead of rewriting files -} - -func isValidFile(f os.FileInfo, extensions []string) bool { - if !f.IsDir() && !strings.HasPrefix(f.Name(), ".") { - for _, ext := range extensions { - if strings.HasSuffix(f.Name(), "."+ext) { - return true - } - } - } - - return false -} - -// If in == nil, the source is the contents of the file with the given filename. -func processFile(filename string, in io.Reader, out io.Writer, stdin bool, opts Options) error { - if in == nil { - f, err := os.Open(filename) - if err != nil { - return err - } - defer f.Close() - in = f - } - - src, err := ioutil.ReadAll(in) - if err != nil { - return err - } - - res, err := printer.Format(src) - if err != nil { - return err - } - // Files should end with newlines - res = append(res, []byte("\n")...) - - if !bytes.Equal(src, res) { - // formatting has changed - if opts.List { - fmt.Fprintln(out, filename) - } - if opts.Write { - err = ioutil.WriteFile(filename, res, 0644) - if err != nil { - return err - } - } - if opts.Diff { - data, err := diff(src, res) - if err != nil { - return fmt.Errorf("computing diff: %s", err) - } - fmt.Fprintf(out, "diff a/%s b/%s\n", filename, filename) - out.Write(data) - } - } - - if !opts.List && !opts.Write && !opts.Diff { - _, err = out.Write(res) - } - - return err -} - -func walkDir(path string, extensions []string, stdout io.Writer, opts Options) error { - visitFile := func(path string, f os.FileInfo, err error) error { - if err == nil && isValidFile(f, extensions) { - err = processFile(path, nil, stdout, false, opts) - } - return err - } - - return filepath.Walk(path, visitFile) -} - -func Run( - paths, extensions []string, - stdin io.Reader, - stdout io.Writer, - opts Options, -) error { - if len(paths) == 0 { - if opts.Write { - return ErrWriteStdin - } - if err := processFile("", stdin, stdout, true, opts); err != nil { - return err - } - return nil - } - - for _, path := range paths { - switch dir, err := os.Stat(path); { - case err != nil: - return err - case dir.IsDir(): - if err := walkDir(path, extensions, stdout, opts); err != nil { - return err - } - default: - if err := processFile(path, nil, stdout, false, opts); err != nil { - return err - } - } - } - - return nil -} - -func diff(b1, b2 []byte) (data []byte, err error) { - f1, err := ioutil.TempFile("", "") - if err != nil { - return - } - defer os.Remove(f1.Name()) - defer f1.Close() - - f2, err := ioutil.TempFile("", "") - if err != nil { - return - } - defer os.Remove(f2.Name()) - defer f2.Close() - - f1.Write(b1) - f2.Write(b2) - - data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput() - if len(data) > 0 { - // diff exits with a non-zero status when the files don't match. - // Ignore that failure as long as we get output. - err = nil - } - return -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd_test.go b/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd_test.go deleted file mode 100644 index 4467b3eb..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/fmtcmd_test.go +++ /dev/null @@ -1,431 +0,0 @@ -package fmtcmd - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "regexp" - "sort" - "syscall" - "testing" -) - -var fixtureExtensions = []string{"hcl"} - -func init() { - sort.Sort(ByFilename(fixtures)) -} - -func TestIsValidFile(t *testing.T) { - const fixtureDir = "./test-fixtures" - - cases := []struct { - Path string - Expected bool - }{ - {"good.hcl", true}, - {".hidden.ignore", false}, - {"file.ignore", false}, - {"dir.ignore", false}, - } - - for _, tc := range cases { - file, err := os.Stat(filepath.Join(fixtureDir, tc.Path)) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - - if res := isValidFile(file, fixtureExtensions); res != tc.Expected { - t.Errorf("want: %b, got: %b", tc.Expected, res) - } - } -} - -func TestRunMultiplePaths(t *testing.T) { - path1, err := renderFixtures("") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(path1) - path2, err := renderFixtures("") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(path2) - - var expectedOut bytes.Buffer - for _, path := range []string{path1, path2} { - for _, fixture := range fixtures { - if !bytes.Equal(fixture.golden, fixture.input) { - expectedOut.WriteString(filepath.Join(path, fixture.filename) + "\n") - } - } - } - - _, stdout := mockIO() - err = Run( - []string{path1, path2}, - fixtureExtensions, - nil, stdout, - Options{ - List: true, - }, - ) - - if err != nil { - t.Errorf("unexpected error: %s", err) - } - if stdout.String() != expectedOut.String() { - t.Errorf("stdout want:\n%s\ngot:\n%s", expectedOut, stdout) - } -} - -func TestRunSubDirectories(t *testing.T) { - pathParent, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(pathParent) - - path1, err := renderFixtures(pathParent) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - path2, err := renderFixtures(pathParent) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - - paths := []string{path1, path2} - sort.Strings(paths) - - var expectedOut bytes.Buffer - for _, path := range paths { - for _, fixture := range fixtures { - if !bytes.Equal(fixture.golden, fixture.input) { - expectedOut.WriteString(filepath.Join(path, fixture.filename) + "\n") - } - } - } - - _, stdout := mockIO() - err = Run( - []string{pathParent}, - fixtureExtensions, - nil, stdout, - Options{ - List: true, - }, - ) - - if err != nil { - t.Errorf("unexpected error: %s", err) - } - if stdout.String() != expectedOut.String() { - t.Errorf("stdout want:\n%s\ngot:\n%s", expectedOut, stdout) - } -} - -func TestRunStdin(t *testing.T) { - var expectedOut bytes.Buffer - for i, fixture := range fixtures { - if i != 0 { - expectedOut.WriteString("\n") - } - expectedOut.Write(fixture.golden) - } - - stdin, stdout := mockIO() - for _, fixture := range fixtures { - stdin.Write(fixture.input) - } - - err := Run( - []string{}, - fixtureExtensions, - stdin, stdout, - Options{}, - ) - - if err != nil { - t.Errorf("unexpected error: %s", err) - } - if !bytes.Equal(stdout.Bytes(), expectedOut.Bytes()) { - t.Errorf("stdout want:\n%s\ngot:\n%s", expectedOut, stdout) - } -} - -func TestRunStdinAndWrite(t *testing.T) { - var expectedOut = []byte{} - - stdin, stdout := mockIO() - stdin.WriteString("") - err := Run( - []string{}, []string{}, - stdin, stdout, - Options{ - Write: true, - }, - ) - - if err != ErrWriteStdin { - t.Errorf("error want:\n%s\ngot:\n%s", ErrWriteStdin, err) - } - if !bytes.Equal(stdout.Bytes(), expectedOut) { - t.Errorf("stdout want:\n%s\ngot:\n%s", expectedOut, stdout) - } -} - -func TestRunFileError(t *testing.T) { - path, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(path) - filename := filepath.Join(path, "unreadable.hcl") - - var expectedError = &os.PathError{ - Op: "open", - Path: filename, - Err: syscall.EACCES, - } - - err = ioutil.WriteFile(filename, []byte{}, 0000) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - - _, stdout := mockIO() - err = Run( - []string{path}, - fixtureExtensions, - nil, stdout, - Options{}, - ) - - if !reflect.DeepEqual(err, expectedError) { - t.Errorf("error want: %#v, got: %#v", expectedError, err) - } -} - -func TestRunNoOptions(t *testing.T) { - path, err := renderFixtures("") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(path) - - var expectedOut bytes.Buffer - for _, fixture := range fixtures { - expectedOut.Write(fixture.golden) - } - - _, stdout := mockIO() - err = Run( - []string{path}, - fixtureExtensions, - nil, stdout, - Options{}, - ) - - if err != nil { - t.Errorf("unexpected error: %s", err) - } - if stdout.String() != expectedOut.String() { - t.Errorf("stdout want:\n%s\ngot:\n%s", expectedOut, stdout) - } -} - -func TestRunList(t *testing.T) { - path, err := renderFixtures("") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(path) - - var expectedOut bytes.Buffer - for _, fixture := range fixtures { - if !bytes.Equal(fixture.golden, fixture.input) { - expectedOut.WriteString(fmt.Sprintln(filepath.Join(path, fixture.filename))) - } - } - - _, stdout := mockIO() - err = Run( - []string{path}, - fixtureExtensions, - nil, stdout, - Options{ - List: true, - }, - ) - - if err != nil { - t.Errorf("unexpected error: %s", err) - } - if stdout.String() != expectedOut.String() { - t.Errorf("stdout want:\n%s\ngot:\n%s", expectedOut, stdout) - } -} - -func TestRunWrite(t *testing.T) { - path, err := renderFixtures("") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(path) - - _, stdout := mockIO() - err = Run( - []string{path}, - fixtureExtensions, - nil, stdout, - Options{ - Write: true, - }, - ) - - if err != nil { - t.Errorf("unexpected error: %s", err) - } - for _, fixture := range fixtures { - res, err := ioutil.ReadFile(filepath.Join(path, fixture.filename)) - if err != nil { - t.Errorf("unexpected error: %s", err) - } - if !bytes.Equal(res, fixture.golden) { - t.Errorf("file %q contents want:\n%s\ngot:\n%s", fixture.filename, fixture.golden, res) - } - } -} - -func TestRunDiff(t *testing.T) { - path, err := renderFixtures("") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - defer os.RemoveAll(path) - - var expectedOut bytes.Buffer - for _, fixture := range fixtures { - if len(fixture.diff) > 0 { - expectedOut.WriteString( - regexp.QuoteMeta( - fmt.Sprintf("diff a/%s/%s b/%s/%s\n", path, fixture.filename, path, fixture.filename), - ), - ) - // Need to use regex to ignore datetimes in diff. - expectedOut.WriteString(`--- .+?\n`) - expectedOut.WriteString(`\+\+\+ .+?\n`) - expectedOut.WriteString(regexp.QuoteMeta(string(fixture.diff))) - } - } - - _, stdout := mockIO() - err = Run( - []string{path}, - fixtureExtensions, - nil, stdout, - Options{ - Diff: true, - }, - ) - - if err != nil { - t.Errorf("unexpected error: %s", err) - } - if !regexp.MustCompile(expectedOut.String()).Match(stdout.Bytes()) { - t.Errorf("stdout want match:\n%s\ngot:\n%q", expectedOut, stdout) - } -} - -func mockIO() (stdin, stdout *bytes.Buffer) { - return new(bytes.Buffer), new(bytes.Buffer) -} - -type fixture struct { - filename string - input, golden, diff []byte -} - -type ByFilename []fixture - -func (s ByFilename) Len() int { return len(s) } -func (s ByFilename) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s ByFilename) Less(i, j int) bool { return len(s[i].filename) > len(s[j].filename) } - -var fixtures = []fixture{ - { - "noop.hcl", - []byte(`resource "aws_security_group" "firewall" { - count = 5 -} -`), - []byte(`resource "aws_security_group" "firewall" { - count = 5 -} -`), - []byte(``), - }, { - "align_equals.hcl", - []byte(`variable "foo" { - default = "bar" - description = "bar" -} -`), - []byte(`variable "foo" { - default = "bar" - description = "bar" -} -`), - []byte(`@@ -1,4 +1,4 @@ - variable "foo" { -- default = "bar" -+ default = "bar" - description = "bar" - } -`), - }, { - "indentation.hcl", - []byte(`provider "aws" { - access_key = "foo" - secret_key = "bar" -} -`), - []byte(`provider "aws" { - access_key = "foo" - secret_key = "bar" -} -`), - []byte(`@@ -1,4 +1,4 @@ - provider "aws" { -- access_key = "foo" -- secret_key = "bar" -+ access_key = "foo" -+ secret_key = "bar" - } -`), - }, -} - -// parent can be an empty string, in which case the system's default -// temporary directory will be used. -func renderFixtures(parent string) (path string, err error) { - path, err = ioutil.TempDir(parent, "") - if err != nil { - return "", err - } - - for _, fixture := range fixtures { - err = ioutil.WriteFile(filepath.Join(path, fixture.filename), []byte(fixture.input), 0644) - if err != nil { - os.RemoveAll(path) - return "", err - } - } - - return path, nil -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/dir.ignore b/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/dir.ignore deleted file mode 100644 index e69de29b..00000000 diff --git a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/file.ignore b/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/file.ignore deleted file mode 100644 index 9977a283..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/file.ignore +++ /dev/null @@ -1 +0,0 @@ -invalid diff --git a/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/good.hcl b/vendor/github.com/hashicorp/hcl/hcl/fmtcmd/test-fixtures/good.hcl deleted file mode 100644 index e69de29b..00000000 diff --git a/vendor/github.com/hashicorp/hcl/hcl/parser/error_test.go b/vendor/github.com/hashicorp/hcl/hcl/parser/error_test.go deleted file mode 100644 index 32399fec..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/parser/error_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package parser - -import ( - "testing" -) - -func TestPosError_impl(t *testing.T) { - var _ error = new(PosError) -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go b/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go index 086c0876..6e54bed9 100644 --- a/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go +++ b/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go @@ -5,6 +5,7 @@ package parser import ( "errors" "fmt" + "strings" "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/hcl/hcl/scanner" @@ -49,7 +50,7 @@ func (p *Parser) Parse() (*ast.File, error) { scerr = &PosError{Pos: pos, Err: errors.New(msg)} } - f.Node, err = p.objectList() + f.Node, err = p.objectList(false) if scerr != nil { return nil, scerr } @@ -61,11 +62,23 @@ func (p *Parser) Parse() (*ast.File, error) { return f, nil } -func (p *Parser) objectList() (*ast.ObjectList, error) { +// objectList parses a list of items within an object (generally k/v pairs). +// The parameter" obj" tells this whether to we are within an object (braces: +// '{', '}') or just at the top level. If we're within an object, we end +// at an RBRACE. +func (p *Parser) objectList(obj bool) (*ast.ObjectList, error) { defer un(trace(p, "ParseObjectList")) node := &ast.ObjectList{} for { + if obj { + tok := p.scan() + p.unscan() + if tok.Type == token.RBRACE { + break + } + } + n, err := p.objectItem() if err == errEofToken { break // we are finished @@ -78,6 +91,13 @@ func (p *Parser) objectList() (*ast.ObjectList, error) { } node.Add(n) + + // object lists can be optionally comma-delimited e.g. when a list of maps + // is being expressed, so a comma is allowed here - it's simply consumed + tok := p.scan() + if tok.Type != token.COMMA { + p.unscan() + } } return node, nil } @@ -122,6 +142,24 @@ func (p *Parser) objectItem() (*ast.ObjectItem, error) { defer un(trace(p, "ParseObjectItem")) keys, err := p.objectKey() + if len(keys) > 0 && err == errEofToken { + // We ignore eof token here since it is an error if we didn't + // receive a value (but we did receive a key) for the item. + err = nil + } + if len(keys) > 0 && err != nil && p.tok.Type == token.RBRACE { + // This is a strange boolean statement, but what it means is: + // We have keys with no value, and we're likely in an object + // (since RBrace ends an object). For this, we set err to nil so + // we continue and get the error below of having the wrong value + // type. + err = nil + + // Reset the token type so we don't think it completed fine. See + // objectType which uses p.tok.Type to check if we're done with + // the object. + p.tok.Type = token.EOF + } if err != nil { return nil, err } @@ -147,6 +185,15 @@ func (p *Parser) objectItem() (*ast.ObjectItem, error) { if err != nil { return nil, err } + default: + keyStr := make([]string, 0, len(keys)) + for _, k := range keys { + keyStr = append(keyStr, k.Token.Text) + } + + return nil, fmt.Errorf( + "key '%s' expected start of object ('{') or assignment ('=')", + strings.Join(keyStr, " ")) } // do a look-ahead for line comment @@ -168,7 +215,11 @@ func (p *Parser) objectKey() ([]*ast.ObjectKey, error) { tok := p.scan() switch tok.Type { case token.EOF: - return nil, errEofToken + // It is very important to also return the keys here as well as + // the error. This is because we need to be able to tell if we + // did parse keys prior to finding the EOF, or if we just found + // a bare EOF. + return keys, errEofToken case token.ASSIGN: // assignment or object only, but not nested objects. this is not // allowed: `foo bar = {}` @@ -188,15 +239,29 @@ func (p *Parser) objectKey() ([]*ast.ObjectKey, error) { return keys, nil case token.LBRACE: + var err error + + // If we have no keys, then it is a syntax error. i.e. {{}} is not + // allowed. + if len(keys) == 0 { + err = &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf("expected: IDENT | STRING got: %s", p.tok.Type), + } + } + // object - return keys, nil + return keys, err case token.IDENT, token.STRING: keyCount++ keys = append(keys, &ast.ObjectKey{Token: p.tok}) case token.ILLEGAL: - fmt.Println("illegal") + return keys, &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf("illegal character"), + } default: - return nil, &PosError{ + return keys, &PosError{ Pos: p.tok.Pos, Err: fmt.Errorf("expected: IDENT | STRING | ASSIGN | LBRACE got: %s", p.tok.Type), } @@ -238,7 +303,7 @@ func (p *Parser) objectType() (*ast.ObjectType, error) { Lbrace: p.tok.Pos, } - l, err := p.objectList() + l, err := p.objectList(true) // if we hit RBRACE, we are good to go (means we parsed all Items), if it's // not a RBRACE, it's an syntax error and we just return it. @@ -246,9 +311,9 @@ func (p *Parser) objectType() (*ast.ObjectType, error) { return nil, err } - // If there is no error, we should be at a RBRACE to end the object - if p.tok.Type != token.RBRACE { - return nil, fmt.Errorf("object expected closing RBRACE got: %s", p.tok.Type) + // No error, scan and expect the ending to be a brace + if tok := p.scan(); tok.Type != token.RBRACE { + return nil, fmt.Errorf("object expected closing RBRACE got: %s", tok.Type) } o.List = l @@ -268,27 +333,38 @@ func (p *Parser) listType() (*ast.ListType, error) { needComma := false for { tok := p.scan() - switch tok.Type { - case token.NUMBER, token.FLOAT, token.STRING, token.HEREDOC: - if needComma { + if needComma { + switch tok.Type { + case token.COMMA, token.RBRACK: + default: return nil, &PosError{ Pos: tok.Pos, - Err: fmt.Errorf("unexpected token: %s. Expecting %s", tok.Type, token.COMMA), + Err: fmt.Errorf( + "error parsing list, expected comma or list end, got: %s", + tok.Type), } } - + } + switch tok.Type { + case token.BOOL, token.NUMBER, token.FLOAT, token.STRING, token.HEREDOC: node, err := p.literalType() if err != nil { return nil, err } + // If there is a lead comment, apply it + if p.leadComment != nil { + node.LeadComment = p.leadComment + p.leadComment = nil + } + l.Add(node) needComma = true case token.COMMA: // get next list item or we are at the end // do a look-ahead for line comment p.scan() - if p.lineComment != nil { + if p.lineComment != nil && len(l.List) > 0 { lit, ok := l.List[len(l.List)-1].(*ast.LiteralType) if ok { lit.LineComment = p.lineComment @@ -300,12 +376,28 @@ func (p *Parser) listType() (*ast.ListType, error) { needComma = false continue - case token.BOOL: - // TODO(arslan) should we support? not supported by HCL yet + case token.LBRACE: + // Looks like a nested object, so parse it out + node, err := p.objectType() + if err != nil { + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf( + "error while trying to parse object within list: %s", err), + } + } + l.Add(node) + needComma = true case token.LBRACK: - // TODO(arslan) should we support nested lists? Even though it's - // written in README of HCL, it's not a part of the grammar - // (not defined in parse.y) + node, err := p.listType() + if err != nil { + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf( + "error while trying to parse list within list: %s", err), + } + } + l.Add(node) case token.RBRACK: // finished l.Rbrack = p.tok.Pos diff --git a/vendor/github.com/hashicorp/hcl/hcl/parser/parser_test.go b/vendor/github.com/hashicorp/hcl/hcl/parser/parser_test.go deleted file mode 100644 index 2ea95c14..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/parser/parser_test.go +++ /dev/null @@ -1,330 +0,0 @@ -package parser - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "reflect" - "runtime" - "testing" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/token" -) - -func TestType(t *testing.T) { - var literals = []struct { - typ token.Type - src string - }{ - {token.STRING, `foo = "foo"`}, - {token.NUMBER, `foo = 123`}, - {token.NUMBER, `foo = -29`}, - {token.FLOAT, `foo = 123.12`}, - {token.FLOAT, `foo = -123.12`}, - {token.BOOL, `foo = true`}, - {token.HEREDOC, "foo = < 0 { - commented = true - buf.WriteByte(newline) - } - - buf.Write(p.indent([]byte(comment.Text))) - buf.WriteByte(newline) - if index != len(o.List.Items) { - buf.WriteByte(newline) // do not print on the end - } - } - } - } - - if index == len(o.List.Items) { - p.prev = o.Rbrace - break - } - - // check if we have adjacent one liner items. If yes we'll going to align - // the comments. - var aligned []*ast.ObjectItem - for _, item := range o.List.Items[index:] { - // we don't group one line lists - if len(o.List.Items) == 1 { - break - } - - // one means a oneliner with out any lead comment - // two means a oneliner with lead comment - // anything else might be something else - cur := lines(string(p.objectItem(item))) - if cur > 2 { - break - } - - curPos := item.Pos() - - nextPos := token.Pos{} - if index != len(o.List.Items)-1 { - nextPos = o.List.Items[index+1].Pos() - } - - prevPos := token.Pos{} - if index != 0 { - prevPos = o.List.Items[index-1].Pos() - } - - // fmt.Println("DEBUG ----------------") - // fmt.Printf("prev = %+v prevPos: %s\n", prev, prevPos) - // fmt.Printf("cur = %+v curPos: %s\n", cur, curPos) - // fmt.Printf("next = %+v nextPos: %s\n", next, nextPos) - - if curPos.Line+1 == nextPos.Line { - aligned = append(aligned, item) - index++ - continue - } - - if curPos.Line-1 == prevPos.Line { - aligned = append(aligned, item) - index++ - - // finish if we have a new line or comment next. This happens - // if the next item is not adjacent - if curPos.Line+1 != nextPos.Line { - break - } - continue - } - - break - } - - // put newlines if the items are between other non aligned items. - // newlines are also added if there is a standalone comment already, so - // check it too - if !commented && index != len(aligned) { - buf.WriteByte(newline) - } - - if len(aligned) >= 1 { - p.prev = aligned[len(aligned)-1].Pos() - - items := p.alignedItems(aligned) - buf.Write(p.indent(items)) - } else { - p.prev = o.List.Items[index].Pos() - - buf.Write(p.indent(p.objectItem(o.List.Items[index]))) - index++ - } - - buf.WriteByte(newline) - } - - buf.WriteString("}") - return buf.Bytes() -} - -func (p *printer) alignedItems(items []*ast.ObjectItem) []byte { - var buf bytes.Buffer - - // find the longest key and value length, needed for alignment - var longestKeyLen int // longest key length - var longestValLen int // longest value length - for _, item := range items { - key := len(item.Keys[0].Token.Text) - val := len(p.output(item.Val)) - - if key > longestKeyLen { - longestKeyLen = key - } - - if val > longestValLen { - longestValLen = val - } - } - - for i, item := range items { - if item.LeadComment != nil { - for _, comment := range item.LeadComment.List { - buf.WriteString(comment.Text) - buf.WriteByte(newline) - } - } - - for i, k := range item.Keys { - keyLen := len(k.Token.Text) - buf.WriteString(k.Token.Text) - for i := 0; i < longestKeyLen-keyLen+1; i++ { - buf.WriteByte(blank) - } - - // reach end of key - if i == len(item.Keys)-1 && len(item.Keys) == 1 { - buf.WriteString("=") - buf.WriteByte(blank) - } - } - - val := p.output(item.Val) - valLen := len(val) - buf.Write(val) - - if item.Val.Pos().Line == item.Keys[0].Pos().Line && item.LineComment != nil { - for i := 0; i < longestValLen-valLen+1; i++ { - buf.WriteByte(blank) - } - - for _, comment := range item.LineComment.List { - buf.WriteString(comment.Text) - } - } - - // do not print for the last item - if i != len(items)-1 { - buf.WriteByte(newline) - } - } - - return buf.Bytes() -} - -// list returns the printable HCL form of an list type. -func (p *printer) list(l *ast.ListType) []byte { - var buf bytes.Buffer - buf.WriteString("[") - - var longestLine int - for _, item := range l.List { - // for now we assume that the list only contains literal types - if lit, ok := item.(*ast.LiteralType); ok { - lineLen := len(lit.Token.Text) - if lineLen > longestLine { - longestLine = lineLen - } - } - } - - insertSpaceBeforeItem := false - for i, item := range l.List { - if item.Pos().Line != l.Lbrack.Line { - // multiline list, add newline before we add each item - buf.WriteByte(newline) - insertSpaceBeforeItem = false - // also indent each line - val := p.output(item) - curLen := len(val) - buf.Write(p.indent(val)) - buf.WriteString(",") - - if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil { - // if the next item doesn't have any comments, do not align - buf.WriteByte(blank) // align one space - for i := 0; i < longestLine-curLen; i++ { - buf.WriteByte(blank) - } - - for _, comment := range lit.LineComment.List { - buf.WriteString(comment.Text) - } - } - - if i == len(l.List)-1 { - buf.WriteByte(newline) - } - } else { - if insertSpaceBeforeItem { - buf.WriteByte(blank) - insertSpaceBeforeItem = false - } - buf.Write(p.output(item)) - if i != len(l.List)-1 { - buf.WriteString(",") - insertSpaceBeforeItem = true - } - } - - } - - buf.WriteString("]") - return buf.Bytes() -} - -// indent indents the lines of the given buffer for each non-empty line -func (p *printer) indent(buf []byte) []byte { - var prefix []byte - if p.cfg.SpacesWidth != 0 { - for i := 0; i < p.cfg.SpacesWidth; i++ { - prefix = append(prefix, blank) - } - } else { - prefix = []byte{tab} - } - - var res []byte - bol := true - for _, c := range buf { - if bol && c != '\n' { - res = append(res, prefix...) - } - - res = append(res, c) - bol = c == '\n' - } - return res -} - -// unindent removes all the indentation from the tombstoned lines -func (p *printer) unindent(buf []byte) []byte { - var res []byte - for i := 0; i < len(buf); i++ { - skip := len(buf)-i <= len(unindent) - if !skip { - skip = !bytes.Equal(unindent, buf[i:i+len(unindent)]) - } - if skip { - res = append(res, buf[i]) - continue - } - - // We have a marker. we have to backtrace here and clean out - // any whitespace ahead of our tombstone up to a \n - for j := len(res) - 1; j >= 0; j-- { - if res[j] == '\n' { - break - } - - res = res[:j] - } - - // Skip the entire unindent marker - i += len(unindent) - 1 - } - - return res -} - -// heredocIndent marks all the 2nd and further lines as unindentable -func (p *printer) heredocIndent(buf []byte) []byte { - var res []byte - bol := false - for _, c := range buf { - if bol && c != '\n' { - res = append(res, unindent...) - } - res = append(res, c) - bol = c == '\n' - } - return res -} - -func lines(txt string) int { - endline := 1 - for i := 0; i < len(txt); i++ { - if txt[i] == '\n' { - endline++ - } - } - return endline -} - -// ---------------------------------------------------------------------------- -// Tracing support - -func (p *printer) printTrace(a ...interface{}) { - if !p.enableTrace { - return - } - - const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " - const n = len(dots) - i := 2 * p.indentTrace - for i > n { - fmt.Print(dots) - i -= n - } - // i <= n - fmt.Print(dots[0:i]) - fmt.Println(a...) -} - -func trace(p *printer, msg string) *printer { - p.printTrace(msg, "(") - p.indentTrace++ - return p -} - -// Usage pattern: defer un(trace(p, "...")) -func un(p *printer) { - p.indentTrace-- - p.printTrace(")") -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go b/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go deleted file mode 100644 index fb9df58d..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go +++ /dev/null @@ -1,64 +0,0 @@ -// Package printer implements printing of AST nodes to HCL format. -package printer - -import ( - "bytes" - "io" - "text/tabwriter" - - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/hcl/hcl/parser" -) - -var DefaultConfig = Config{ - SpacesWidth: 2, -} - -// A Config node controls the output of Fprint. -type Config struct { - SpacesWidth int // if set, it will use spaces instead of tabs for alignment -} - -func (c *Config) Fprint(output io.Writer, node ast.Node) error { - p := &printer{ - cfg: *c, - comments: make([]*ast.CommentGroup, 0), - standaloneComments: make([]*ast.CommentGroup, 0), - // enableTrace: true, - } - - p.collectComments(node) - - if _, err := output.Write(p.unindent(p.output(node))); err != nil { - return err - } - - // flush tabwriter, if any - var err error - if tw, _ := output.(*tabwriter.Writer); tw != nil { - err = tw.Flush() - } - - return err -} - -// Fprint "pretty-prints" an HCL node to output -// It calls Config.Fprint with default settings. -func Fprint(output io.Writer, node ast.Node) error { - return DefaultConfig.Fprint(output, node) -} - -// Format formats src HCL and returns the result. -func Format(src []byte) ([]byte, error) { - node, err := parser.Parse(src) - if err != nil { - return nil, err - } - - var buf bytes.Buffer - if err := DefaultConfig.Fprint(&buf, node); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/printer_test.go b/vendor/github.com/hashicorp/hcl/hcl/printer/printer_test.go deleted file mode 100644 index 86aa946b..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/printer_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package printer - -import ( - "bytes" - "errors" - "flag" - "fmt" - "io/ioutil" - "path/filepath" - "testing" - - "github.com/hashicorp/hcl/hcl/parser" -) - -var update = flag.Bool("update", false, "update golden files") - -const ( - dataDir = "testdata" -) - -type entry struct { - source, golden string -} - -// Use go test -update to create/update the respective golden files. -var data = []entry{ - {"complexhcl.input", "complexhcl.golden"}, - {"list.input", "list.golden"}, - {"comment.input", "comment.golden"}, - {"comment_aligned.input", "comment_aligned.golden"}, - {"comment_standalone.input", "comment_standalone.golden"}, -} - -func TestFiles(t *testing.T) { - for _, e := range data { - source := filepath.Join(dataDir, e.source) - golden := filepath.Join(dataDir, e.golden) - check(t, source, golden) - } -} - -func check(t *testing.T, source, golden string) { - src, err := ioutil.ReadFile(source) - if err != nil { - t.Error(err) - return - } - - res, err := format(src) - if err != nil { - t.Error(err) - return - } - - // update golden files if necessary - if *update { - if err := ioutil.WriteFile(golden, res, 0644); err != nil { - t.Error(err) - } - return - } - - // get golden - gld, err := ioutil.ReadFile(golden) - if err != nil { - t.Error(err) - return - } - - // formatted source and golden must be the same - if err := diff(source, golden, res, gld); err != nil { - t.Error(err) - return - } -} - -// diff compares a and b. -func diff(aname, bname string, a, b []byte) error { - var buf bytes.Buffer // holding long error message - - // compare lengths - if len(a) != len(b) { - fmt.Fprintf(&buf, "\nlength changed: len(%s) = %d, len(%s) = %d", aname, len(a), bname, len(b)) - } - - // compare contents - line := 1 - offs := 1 - for i := 0; i < len(a) && i < len(b); i++ { - ch := a[i] - if ch != b[i] { - fmt.Fprintf(&buf, "\n%s:%d:%d: %s", aname, line, i-offs+1, lineAt(a, offs)) - fmt.Fprintf(&buf, "\n%s:%d:%d: %s", bname, line, i-offs+1, lineAt(b, offs)) - fmt.Fprintf(&buf, "\n\n") - break - } - if ch == '\n' { - line++ - offs = i + 1 - } - } - - if buf.Len() > 0 { - return errors.New(buf.String()) - } - return nil -} - -// format parses src, prints the corresponding AST, verifies the resulting -// src is syntactically correct, and returns the resulting src or an error -// if any. -func format(src []byte) ([]byte, error) { - // parse src - node, err := parser.Parse(src) - if err != nil { - return nil, fmt.Errorf("parse: %s\n%s", err, src) - } - - var buf bytes.Buffer - - cfg := &Config{} - if err := cfg.Fprint(&buf, node); err != nil { - return nil, fmt.Errorf("print: %s", err) - } - - // make sure formatted output is syntactically correct - res := buf.Bytes() - - if _, err := parser.Parse(src); err != nil { - return nil, fmt.Errorf("parse: %s\n%s", err, src) - } - - return res, nil -} - -// lineAt returns the line in text starting at offset offs. -func lineAt(text []byte, offs int) []byte { - i := offs - for i < len(text) && text[i] != '\n' { - i++ - } - return text[offs:i] -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden b/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden deleted file mode 100644 index e86215f5..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.golden +++ /dev/null @@ -1,36 +0,0 @@ -// A standalone comment is a comment which is not attached to any kind of node - -// This comes from Terraform, as a test -variable "foo" { - # Standalone comment should be still here - - default = "bar" - description = "bar" # yooo -} - -/* This is a multi line standalone -comment*/ - -// fatih arslan -/* This is a developer test -account and a multine comment */ -developer = ["fatih", "arslan"] // fatih arslan - -# One line here -numbers = [1, 2] // another line here - -# Another comment -variable = { - description = "bar" # another yooo - - foo { - # Nested standalone - - bar = "fatih" - } -} - -// lead comment -foo { - bar = "fatih" // line comment 2 -} // line comment 3 \ No newline at end of file diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input b/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input deleted file mode 100644 index 57c37ac1..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment.input +++ /dev/null @@ -1,37 +0,0 @@ -// A standalone comment is a comment which is not attached to any kind of node - - // This comes from Terraform, as a test -variable "foo" { - # Standalone comment should be still here - - default = "bar" - description = "bar" # yooo -} - -/* This is a multi line standalone -comment*/ - - -// fatih arslan -/* This is a developer test -account and a multine comment */ -developer = [ "fatih", "arslan"] // fatih arslan - -# One line here -numbers = [1,2] // another line here - - # Another comment -variable = { - description = "bar" # another yooo - foo { - # Nested standalone - - bar = "fatih" - } -} - - // lead comment -foo { - bar = "fatih" // line comment 2 -} // line comment 3 - diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden b/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden deleted file mode 100644 index e8469e5c..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.golden +++ /dev/null @@ -1,32 +0,0 @@ -aligned { - # We have some aligned items below - foo = "fatih" # yoo1 - default = "bar" # yoo2 - bar = "bar and foo" # yoo3 - - default = { - bar = "example" - } - - #deneme arslan - fatih = ["fatih"] # yoo4 - - #fatih arslan - fatiharslan = ["arslan"] // yoo5 - - default = { - bar = "example" - } - - security_groups = [ - "foo", # kenya 1 - "${aws_security_group.firewall.foo}", # kenya 2 - ] - - security_groups2 = [ - "foo", # kenya 1 - "bar", # kenya 1.5 - "${aws_security_group.firewall.foo}", # kenya 2 - "foobar", # kenya 3 - ] -} \ No newline at end of file diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input b/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input deleted file mode 100644 index bd43ab1a..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_aligned.input +++ /dev/null @@ -1,28 +0,0 @@ -aligned { -# We have some aligned items below - foo = "fatih" # yoo1 - default = "bar" # yoo2 - bar = "bar and foo" # yoo3 - default = { - bar = "example" - } - #deneme arslan - fatih = ["fatih"] # yoo4 - #fatih arslan - fatiharslan = ["arslan"] // yoo5 - default = { - bar = "example" - } - -security_groups = [ - "foo", # kenya 1 - "${aws_security_group.firewall.foo}", # kenya 2 -] - -security_groups2 = [ - "foo", # kenya 1 - "bar", # kenya 1.5 - "${aws_security_group.firewall.foo}", # kenya 2 - "foobar", # kenya 3 -] -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden b/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden deleted file mode 100644 index 962dbf2b..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.golden +++ /dev/null @@ -1,16 +0,0 @@ -// A standalone comment - -aligned { - # Standalone 1 - - a = "bar" # yoo1 - default = "bar" # yoo2 - - # Standalone 2 -} - -# Standalone 3 - -numbers = [1, 2] // another line here - -# Standalone 4 diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input b/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input deleted file mode 100644 index 4436cb16..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/comment_standalone.input +++ /dev/null @@ -1,16 +0,0 @@ -// A standalone comment - -aligned { - # Standalone 1 - - a = "bar" # yoo1 - default = "bar" # yoo2 - - # Standalone 2 -} - - # Standalone 3 - -numbers = [1,2] // another line here - - # Standalone 4 diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden b/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden deleted file mode 100644 index b733a27e..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/printer/testdata/complexhcl.golden +++ /dev/null @@ -1,54 +0,0 @@ -variable "foo" { - default = "bar" - description = "bar" -} - -developer = ["fatih", "arslan"] - -provider "aws" { - access_key = "foo" - secret_key = "bar" -} - -provider "do" { - api_key = "${var.foo}" -} - -resource "aws_security_group" "firewall" { - count = 5 -} - -resource aws_instance "web" { - ami = "${var.foo}" - - security_groups = [ - "foo", - "${aws_security_group.firewall.foo}", - ] - - network_interface { - device_index = 0 - description = "Main network interface" - } - - network_interface = { - device_index = 1 - - description = < 0 { + s.err("unexpected null character (0x00)") + return eof + } + // debug // fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column) return ch @@ -223,6 +230,11 @@ func (s *Scanner) Scan() token.Token { func (s *Scanner) scanComment(ch rune) { // single line comments if ch == '#' || (ch == '/' && s.peek() != '*') { + if ch == '/' && s.peek() != '/' { + s.err("expected '/' for comment") + return + } + ch = s.next() for ch != '\n' && ch >= 0 && ch != eof { ch = s.next() @@ -376,7 +388,7 @@ func (s *Scanner) scanExponent(ch rune) rune { return ch } -// scanHeredoc scans a heredoc string. +// scanHeredoc scans a heredoc string func (s *Scanner) scanHeredoc() { // Scan the second '<' in example: '<= len(identBytes) && identRegexp.Match(s.src[lineStart:s.srcPos.Offset-s.lastCharLen]) { break } @@ -452,7 +480,7 @@ func (s *Scanner) scanString() { // read character after quote ch := s.next() - if ch == '\n' || ch < 0 || ch == eof { + if (ch == '\n' && braces == 0) || ch < 0 || ch == eof { s.err("literal not terminated") return } @@ -508,16 +536,27 @@ func (s *Scanner) scanEscape() rune { // scanDigits scans a rune with the given base for n times. For example an // octal notation \184 would yield in scanDigits(ch, 8, 3) func (s *Scanner) scanDigits(ch rune, base, n int) rune { + start := n for n > 0 && digitVal(ch) < base { ch = s.next() + if ch == eof { + // If we see an EOF, we halt any more scanning of digits + // immediately. + break + } + n-- } if n > 0 { s.err("illegal char escape") } - // we scanned all digits, put the last non digit char back - s.unread() + if n != start { + // we scanned all digits, put the last non digit char back, + // only if we read anything at all + s.unread() + } + return ch } diff --git a/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner_test.go b/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner_test.go deleted file mode 100644 index 8abcbf34..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner_test.go +++ /dev/null @@ -1,536 +0,0 @@ -package scanner - -import ( - "bytes" - "fmt" - "testing" - - "github.com/hashicorp/hcl/hcl/token" - "strings" -) - -var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" - -type tokenPair struct { - tok token.Type - text string -} - -var tokenLists = map[string][]tokenPair{ - "comment": []tokenPair{ - {token.COMMENT, "//"}, - {token.COMMENT, "////"}, - {token.COMMENT, "// comment"}, - {token.COMMENT, "// /* comment */"}, - {token.COMMENT, "// // comment //"}, - {token.COMMENT, "//" + f100}, - {token.COMMENT, "#"}, - {token.COMMENT, "##"}, - {token.COMMENT, "# comment"}, - {token.COMMENT, "# /* comment */"}, - {token.COMMENT, "# # comment #"}, - {token.COMMENT, "#" + f100}, - {token.COMMENT, "/**/"}, - {token.COMMENT, "/***/"}, - {token.COMMENT, "/* comment */"}, - {token.COMMENT, "/* // comment */"}, - {token.COMMENT, "/* /* comment */"}, - {token.COMMENT, "/*\n comment\n*/"}, - {token.COMMENT, "/*" + f100 + "*/"}, - }, - "operator": []tokenPair{ - {token.LBRACK, "["}, - {token.LBRACE, "{"}, - {token.COMMA, ","}, - {token.PERIOD, "."}, - {token.RBRACK, "]"}, - {token.RBRACE, "}"}, - {token.ASSIGN, "="}, - {token.ADD, "+"}, - {token.SUB, "-"}, - }, - "bool": []tokenPair{ - {token.BOOL, "true"}, - {token.BOOL, "false"}, - }, - "ident": []tokenPair{ - {token.IDENT, "a"}, - {token.IDENT, "a0"}, - {token.IDENT, "foobar"}, - {token.IDENT, "foo-bar"}, - {token.IDENT, "abc123"}, - {token.IDENT, "LGTM"}, - {token.IDENT, "_"}, - {token.IDENT, "_abc123"}, - {token.IDENT, "abc123_"}, - {token.IDENT, "_abc_123_"}, - {token.IDENT, "_äöü"}, - {token.IDENT, "_本"}, - {token.IDENT, "äöü"}, - {token.IDENT, "本"}, - {token.IDENT, "a۰۱۸"}, - {token.IDENT, "foo६४"}, - {token.IDENT, "bar９８７６"}, - }, - "heredoc": []tokenPair{ - {token.HEREDOC, "< 0 for %q", s.ErrorCount, src) - } -} - -func testTokenList(t *testing.T, tokenList []tokenPair) { - // create artifical source code - buf := new(bytes.Buffer) - for _, ident := range tokenList { - fmt.Fprintf(buf, "%s\n", ident.text) - } - - s := New(buf.Bytes()) - for _, ident := range tokenList { - tok := s.Scan() - if tok.Type != ident.tok { - t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text) - } - - if tok.Text != ident.text { - t.Errorf("text = %q want %q", tok.String(), ident.text) - } - - } -} - -func countNewlines(s string) int { - n := 0 - for _, ch := range s { - if ch == '\n' { - n++ - } - } - return n -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/strconv/quote.go b/vendor/github.com/hashicorp/hcl/hcl/strconv/quote.go index e87ac635..5f981eaa 100644 --- a/vendor/github.com/hashicorp/hcl/hcl/strconv/quote.go +++ b/vendor/github.com/hashicorp/hcl/hcl/strconv/quote.go @@ -27,7 +27,7 @@ func Unquote(s string) (t string, err error) { if quote != '"' { return "", ErrSyntax } - if contains(s, '\n') { + if !contains(s, '$') && !contains(s, '{') && contains(s, '\n') { return "", ErrSyntax } @@ -49,7 +49,7 @@ func Unquote(s string) (t string, err error) { for len(s) > 0 { // If we're starting a '${}' then let it through un-unquoted. // Specifically: we don't unquote any characters within the `${}` - // section, except for escaped quotes, which we handle specifically. + // section. if s[0] == '$' && len(s) > 1 && s[1] == '{' { buf = append(buf, '$', '{') s = s[2:] @@ -64,14 +64,6 @@ func Unquote(s string) (t string, err error) { s = s[size:] - // We special case escaped double quotes in interpolations, converting - // them to straight double quotes. - if r == '\\' { - if q, _ := utf8.DecodeRuneInString(s); q == '"' { - continue - } - } - n := utf8.EncodeRune(runeTmp[:], r) buf = append(buf, runeTmp[:n]...) @@ -95,6 +87,10 @@ func Unquote(s string) (t string, err error) { } } + if s[0] == '\n' { + return "", ErrSyntax + } + c, multibyte, ss, err := unquoteChar(s, quote) if err != nil { return "", err diff --git a/vendor/github.com/hashicorp/hcl/hcl/strconv/quote_test.go b/vendor/github.com/hashicorp/hcl/hcl/strconv/quote_test.go deleted file mode 100644 index 4a810aa3..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/strconv/quote_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package strconv - -import "testing" - -type quoteTest struct { - in string - out string - ascii string -} - -var quotetests = []quoteTest{ - {"\a\b\f\r\n\t\v", `"\a\b\f\r\n\t\v"`, `"\a\b\f\r\n\t\v"`}, - {"\\", `"\\"`, `"\\"`}, - {"abc\xffdef", `"abc\xffdef"`, `"abc\xffdef"`}, - {"\u263a", `"☺"`, `"\u263a"`}, - {"\U0010ffff", `"\U0010ffff"`, `"\U0010ffff"`}, - {"\x04", `"\x04"`, `"\x04"`}, -} - -type unQuoteTest struct { - in string - out string -} - -var unquotetests = []unQuoteTest{ - {`""`, ""}, - {`"a"`, "a"}, - {`"abc"`, "abc"}, - {`"☺"`, "☺"}, - {`"hello world"`, "hello world"}, - {`"\xFF"`, "\xFF"}, - {`"\377"`, "\377"}, - {`"\u1234"`, "\u1234"}, - {`"\U00010111"`, "\U00010111"}, - {`"\U0001011111"`, "\U0001011111"}, - {`"\a\b\f\n\r\t\v\\\""`, "\a\b\f\n\r\t\v\\\""}, - {`"'"`, "'"}, - {`"${file("foo")}"`, `${file("foo")}`}, - {`"${file(\"foo\")}"`, `${file("foo")}`}, - {`"echo ${var.region}${element(split(",",var.zones),0)}"`, - `echo ${var.region}${element(split(",",var.zones),0)}`}, -} - -var misquoted = []string{ - ``, - `"`, - `"a`, - `"'`, - `b"`, - `"\"`, - `"\9"`, - `"\19"`, - `"\129"`, - `'\'`, - `'\9'`, - `'\19'`, - `'\129'`, - `'ab'`, - `"\x1!"`, - `"\U12345678"`, - `"\z"`, - "`", - "`xxx", - "`\"", - `"\'"`, - `'\"'`, - "\"\n\"", - "\"\\n\n\"", - "'\n'", - `"${"`, - `"${foo{}"`, -} - -func TestUnquote(t *testing.T) { - for _, tt := range unquotetests { - if out, err := Unquote(tt.in); err != nil || out != tt.out { - t.Errorf("Unquote(%#q) = %q, %v want %q, nil", tt.in, out, err, tt.out) - } - } - - // run the quote tests too, backward - for _, tt := range quotetests { - if in, err := Unquote(tt.out); in != tt.in { - t.Errorf("Unquote(%#q) = %q, %v, want %q, nil", tt.out, in, err, tt.in) - } - } - - for _, s := range misquoted { - if out, err := Unquote(s); out != "" || err != ErrSyntax { - t.Errorf("Unquote(%#q) = %q, %v want %q, %v", s, out, err, "", ErrSyntax) - } - } -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/array_comment.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/array_comment.hcl deleted file mode 100644 index 78c26758..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/array_comment.hcl +++ /dev/null @@ -1,4 +0,0 @@ -foo = [ - "1", - "2", # comment -] diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_colon.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_colon.hcl deleted file mode 100644 index eb5a99a6..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_colon.hcl +++ /dev/null @@ -1,6 +0,0 @@ -resource = [{ - "foo": { - "bar": {}, - "baz": [1, 2, "foo"], - } -}] diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_deep.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_deep.hcl deleted file mode 100644 index dd3151cb..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/assign_deep.hcl +++ /dev/null @@ -1,5 +0,0 @@ -resource = [{ - foo = [{ - bar = {} - }] -}] diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment.hcl deleted file mode 100644 index 1ff7f29f..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment.hcl +++ /dev/null @@ -1,15 +0,0 @@ -// Foo - -/* Bar */ - -/* -/* -Baz -*/ - -# Another - -# Multiple -# Lines - -foo = "bar" diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment_single.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment_single.hcl deleted file mode 100644 index fec56017..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/comment_single.hcl +++ /dev/null @@ -1 +0,0 @@ -# Hello diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex.hcl deleted file mode 100644 index cccb5b06..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex.hcl +++ /dev/null @@ -1,42 +0,0 @@ -// This comes from Terraform, as a test -variable "foo" { - default = "bar" - description = "bar" -} - -provider "aws" { - access_key = "foo" - secret_key = "bar" -} - -provider "do" { - api_key = "${var.foo}" -} - -resource "aws_security_group" "firewall" { - count = 5 -} - -resource aws_instance "web" { - ami = "${var.foo}" - security_groups = [ - "foo", - "${aws_security_group.firewall.foo}" - ] - - network_interface { - device_index = 0 - description = "Main network interface" - } -} - -resource "aws_instance" "db" { - security_groups = "${aws_security_group.firewall.*.id}" - VPC = "foo" - - depends_on = ["aws_instance.web"] -} - -output "web_ip" { - value = "${aws_instance.web.private_ip}" -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex_key.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex_key.hcl deleted file mode 100644 index 0007aaf5..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/complex_key.hcl +++ /dev/null @@ -1 +0,0 @@ -foo.bar = "baz" diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/empty.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/empty.hcl deleted file mode 100644 index e69de29b..00000000 diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list.hcl deleted file mode 100644 index 059d4ce6..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list.hcl +++ /dev/null @@ -1 +0,0 @@ -foo = [1, 2, "foo"] diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list_comma.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list_comma.hcl deleted file mode 100644 index 50f4218a..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/list_comma.hcl +++ /dev/null @@ -1 +0,0 @@ -foo = [1, 2, "foo",] diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/multiple.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/multiple.hcl deleted file mode 100644 index 029c54b0..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/multiple.hcl +++ /dev/null @@ -1,2 +0,0 @@ -foo = "bar" -key = 7 diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/old.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/old.hcl deleted file mode 100644 index e9f77cae..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/old.hcl +++ /dev/null @@ -1,3 +0,0 @@ -default = { - "eu-west-1": "ami-b1cf19c6", -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure.hcl deleted file mode 100644 index 92592fbb..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure.hcl +++ /dev/null @@ -1,5 +0,0 @@ -// This is a test structure for the lexer -foo bar "baz" { - key = 7 - foo = "bar" -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_basic.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_basic.hcl deleted file mode 100644 index 7229a1f0..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_basic.hcl +++ /dev/null @@ -1,5 +0,0 @@ -foo { - value = 7 - "value" = 8 - "complex::value" = 9 -} diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_empty.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_empty.hcl deleted file mode 100644 index 4d156dde..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/structure_empty.hcl +++ /dev/null @@ -1 +0,0 @@ -resource "foo" "bar" {} diff --git a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/types.hcl b/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/types.hcl deleted file mode 100644 index cf2747ea..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/test-fixtures/types.hcl +++ /dev/null @@ -1,7 +0,0 @@ -foo = "bar" -bar = 7 -baz = [1,2,3] -foo = -12 -bar = 3.14159 -foo = true -bar = false diff --git a/vendor/github.com/hashicorp/hcl/hcl/token/token.go b/vendor/github.com/hashicorp/hcl/hcl/token/token.go index 696ee8da..e37c0664 100644 --- a/vendor/github.com/hashicorp/hcl/hcl/token/token.go +++ b/vendor/github.com/hashicorp/hcl/hcl/token/token.go @@ -142,13 +142,7 @@ func (t Token) Value() interface{} { case IDENT: return t.Text case HEREDOC: - // We need to find the end of the marker - idx := strings.IndexByte(t.Text, '\n') - if idx == -1 { - panic("heredoc doesn't contain newline") - } - - return string(t.Text[idx+1 : len(t.Text)-idx+1]) + return unindentHeredoc(t.Text) case STRING: // Determine the Unquote method to use. If it came from JSON, // then we need to use the built-in unquote since we have to @@ -158,6 +152,11 @@ func (t Token) Value() interface{} { f = strconv.Unquote } + // This case occurs if json null is used + if t.Text == "" { + return "" + } + v, err := f(t.Text) if err != nil { panic(fmt.Sprintf("unquote %s err: %s", t.Text, err)) @@ -168,3 +167,53 @@ func (t Token) Value() interface{} { panic(fmt.Sprintf("unimplemented Value for type: %s", t.Type)) } } + +// unindentHeredoc returns the string content of a HEREDOC if it is started with << +// and the content of a HEREDOC with the hanging indent removed if it is started with +// a <<-, and the terminating line is at least as indented as the least indented line. +func unindentHeredoc(heredoc string) string { + // We need to find the end of the marker + idx := strings.IndexByte(heredoc, '\n') + if idx == -1 { + panic("heredoc doesn't contain newline") + } + + unindent := heredoc[2] == '-' + + // We can optimize if the heredoc isn't marked for indentation + if !unindent { + return string(heredoc[idx+1 : len(heredoc)-idx+1]) + } + + // We need to unindent each line based on the indentation level of the marker + lines := strings.Split(string(heredoc[idx+1:len(heredoc)-idx+2]), "\n") + whitespacePrefix := lines[len(lines)-1] + + isIndented := true + for _, v := range lines { + if strings.HasPrefix(v, whitespacePrefix) { + continue + } + + isIndented = false + break + } + + // If all lines are not at least as indented as the terminating mark, return the + // heredoc as is, but trim the leading space from the marker on the final line. + if !isIndented { + return strings.TrimRight(string(heredoc[idx+1:len(heredoc)-idx+1]), " \t") + } + + unindentedLines := make([]string, len(lines)) + for k, v := range lines { + if k == len(lines)-1 { + unindentedLines[k] = "" + break + } + + unindentedLines[k] = strings.TrimPrefix(v, whitespacePrefix) + } + + return strings.Join(unindentedLines, "\n") +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/token/token_test.go b/vendor/github.com/hashicorp/hcl/hcl/token/token_test.go deleted file mode 100644 index b5b766c1..00000000 --- a/vendor/github.com/hashicorp/hcl/hcl/token/token_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package token - -import ( - "reflect" - "testing" -) - -func TestTypeString(t *testing.T) { - var tokens = []struct { - tt Type - str string - }{ - {ILLEGAL, "ILLEGAL"}, - {EOF, "EOF"}, - {COMMENT, "COMMENT"}, - {IDENT, "IDENT"}, - {NUMBER, "NUMBER"}, - {FLOAT, "FLOAT"}, - {BOOL, "BOOL"}, - {STRING, "STRING"}, - {HEREDOC, "HEREDOC"}, - {LBRACK, "LBRACK"}, - {LBRACE, "LBRACE"}, - {COMMA, "COMMA"}, - {PERIOD, "PERIOD"}, - {RBRACK, "RBRACK"}, - {RBRACE, "RBRACE"}, - {ASSIGN, "ASSIGN"}, - {ADD, "ADD"}, - {SUB, "SUB"}, - } - - for _, token := range tokens { - if token.tt.String() != token.str { - t.Errorf("want: %q got:%q\n", token.str, token.tt) - } - } - -} - -func TestTokenValue(t *testing.T) { - var tokens = []struct { - tt Token - v interface{} - }{ - {Token{Type: BOOL, Text: `true`}, true}, - {Token{Type: BOOL, Text: `false`}, false}, - {Token{Type: FLOAT, Text: `3.14`}, float64(3.14)}, - {Token{Type: NUMBER, Text: `42`}, int64(42)}, - {Token{Type: IDENT, Text: `foo`}, "foo"}, - {Token{Type: STRING, Text: `"foo"`}, "foo"}, - {Token{Type: STRING, Text: `"foo\nbar"`}, "foo\nbar"}, - {Token{Type: STRING, Text: `"${file(\"foo\")}"`}, `${file("foo")}`}, - {Token{Type: HEREDOC, Text: "< 0 for %q", s.ErrorCount, src) - } -} - -func testTokenList(t *testing.T, tokenList []tokenPair) { - // create artifical source code - buf := new(bytes.Buffer) - for _, ident := range tokenList { - fmt.Fprintf(buf, "%s\n", ident.text) - } - - s := New(buf.Bytes()) - for _, ident := range tokenList { - tok := s.Scan() - if tok.Type != ident.tok { - t.Errorf("tok = %q want %q for %q\n", tok, ident.tok, ident.text) - } - - if tok.Text != ident.text { - t.Errorf("text = %q want %q", tok.String(), ident.text) - } - - } -} - -func countNewlines(s string) int { - n := 0 - for _, ch := range s { - if ch == '\n' { - n++ - } - } - return n -} diff --git a/vendor/github.com/hashicorp/hcl/json/test-fixtures/array.json b/vendor/github.com/hashicorp/hcl/json/test-fixtures/array.json deleted file mode 100644 index e320f17a..00000000 --- a/vendor/github.com/hashicorp/hcl/json/test-fixtures/array.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "foo": [1, 2, "bar"], - "bar": "baz" -} diff --git a/vendor/github.com/hashicorp/hcl/json/test-fixtures/basic.json b/vendor/github.com/hashicorp/hcl/json/test-fixtures/basic.json deleted file mode 100644 index b54bde96..00000000 --- a/vendor/github.com/hashicorp/hcl/json/test-fixtures/basic.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "foo": "bar" -} diff --git a/vendor/github.com/hashicorp/hcl/json/test-fixtures/object.json b/vendor/github.com/hashicorp/hcl/json/test-fixtures/object.json deleted file mode 100644 index 72168a3c..00000000 --- a/vendor/github.com/hashicorp/hcl/json/test-fixtures/object.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "foo": { - "bar": [1,2] - } -} diff --git a/vendor/github.com/hashicorp/hcl/json/test-fixtures/types.json b/vendor/github.com/hashicorp/hcl/json/test-fixtures/types.json deleted file mode 100644 index 9a142a6c..00000000 --- a/vendor/github.com/hashicorp/hcl/json/test-fixtures/types.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "foo": "bar", - "bar": 7, - "baz": [1,2,3], - "foo": -12, - "bar": 3.14159, - "foo": true, - "bar": false, - "foo": null -} diff --git a/vendor/github.com/hashicorp/hcl/json/token/token_test.go b/vendor/github.com/hashicorp/hcl/json/token/token_test.go deleted file mode 100644 index a83fdd55..00000000 --- a/vendor/github.com/hashicorp/hcl/json/token/token_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package token - -import ( - "testing" -) - -func TestTypeString(t *testing.T) { - var tokens = []struct { - tt Type - str string - }{ - {ILLEGAL, "ILLEGAL"}, - {EOF, "EOF"}, - {NUMBER, "NUMBER"}, - {FLOAT, "FLOAT"}, - {BOOL, "BOOL"}, - {STRING, "STRING"}, - {NULL, "NULL"}, - {LBRACK, "LBRACK"}, - {LBRACE, "LBRACE"}, - {COMMA, "COMMA"}, - {PERIOD, "PERIOD"}, - {RBRACK, "RBRACK"}, - {RBRACE, "RBRACE"}, - } - - for _, token := range tokens { - if token.tt.String() != token.str { - t.Errorf("want: %q got:%q\n", token.str, token.tt) - - } - } - -} diff --git a/vendor/github.com/hashicorp/hcl/lex.go b/vendor/github.com/hashicorp/hcl/lex.go index 2e38ecb0..d9993c29 100644 --- a/vendor/github.com/hashicorp/hcl/lex.go +++ b/vendor/github.com/hashicorp/hcl/lex.go @@ -2,6 +2,7 @@ package hcl import ( "unicode" + "unicode/utf8" ) type lexModeValue byte @@ -14,17 +15,23 @@ const ( // lexMode returns whether we're going to be parsing in JSON // mode or HCL mode. -func lexMode(v string) lexModeValue { - for _, r := range v { +func lexMode(v []byte) lexModeValue { + var ( + r rune + w int + offset int + ) + + for { + r, w = utf8.DecodeRune(v[offset:]) + offset += w if unicode.IsSpace(r) { continue } - if r == '{' { return lexModeJson - } else { - return lexModeHcl } + break } return lexModeHcl diff --git a/vendor/github.com/hashicorp/hcl/lex_test.go b/vendor/github.com/hashicorp/hcl/lex_test.go deleted file mode 100644 index f7ee3788..00000000 --- a/vendor/github.com/hashicorp/hcl/lex_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package hcl - -import ( - "testing" -) - -func TestLexMode(t *testing.T) { - cases := []struct { - Input string - Mode lexModeValue - }{ - { - "", - lexModeHcl, - }, - { - "foo", - lexModeHcl, - }, - { - "{}", - lexModeJson, - }, - { - " {}", - lexModeJson, - }, - } - - for i, tc := range cases { - actual := lexMode(tc.Input) - - if actual != tc.Mode { - t.Fatalf("%d: %#v", i, actual) - } - } -} diff --git a/vendor/github.com/hashicorp/hcl/parse.go b/vendor/github.com/hashicorp/hcl/parse.go index d0719c2a..1fca53c4 100644 --- a/vendor/github.com/hashicorp/hcl/parse.go +++ b/vendor/github.com/hashicorp/hcl/parse.go @@ -8,16 +8,32 @@ import ( jsonParser "github.com/hashicorp/hcl/json/parser" ) -// Parse parses the given input and returns the root object. +// ParseBytes accepts as input byte slice and returns ast tree. // -// The input format can be either HCL or JSON. -func Parse(input string) (*ast.File, error) { - switch lexMode(input) { +// Input can be either JSON or HCL +func ParseBytes(in []byte) (*ast.File, error) { + return parse(in) +} + +// ParseString accepts input as a string and returns ast tree. +func ParseString(input string) (*ast.File, error) { + return parse([]byte(input)) +} + +func parse(in []byte) (*ast.File, error) { + switch lexMode(in) { case lexModeHcl: - return hclParser.Parse([]byte(input)) + return hclParser.Parse(in) case lexModeJson: - return jsonParser.Parse([]byte(input)) + return jsonParser.Parse(in) } return nil, fmt.Errorf("unknown config format") } + +// Parse parses the given input and returns the root object. +// +// The input format can be either HCL or JSON. +func Parse(input string) (*ast.File, error) { + return parse([]byte(input)) +} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/basic.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/basic.hcl deleted file mode 100644 index 94999448..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/basic.hcl +++ /dev/null @@ -1,2 +0,0 @@ -foo = "bar" -bar = "${file("bing/bong.txt")}" diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/basic.json b/vendor/github.com/hashicorp/hcl/test-fixtures/basic.json deleted file mode 100644 index 7bdddc84..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/basic.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "foo": "bar", - "bar": "${file(\"bing/bong.txt\")}" -} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/basic_int_string.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/basic_int_string.hcl deleted file mode 100644 index 4e415da2..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/basic_int_string.hcl +++ /dev/null @@ -1 +0,0 @@ -count = "3" diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/basic_squish.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/basic_squish.hcl deleted file mode 100644 index 363697b4..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/basic_squish.hcl +++ /dev/null @@ -1,3 +0,0 @@ -foo="bar" -bar="${file("bing/bong.txt")}" -foo-bar="baz" diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.hcl deleted file mode 100644 index 5b185cc9..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.hcl +++ /dev/null @@ -1,15 +0,0 @@ -key "" { - policy = "read" -} - -key "foo/" { - policy = "write" -} - -key "foo/bar/" { - policy = "read" -} - -key "foo/bar/baz" { - policy = "deny" -} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.json b/vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.json deleted file mode 100644 index 151864ee..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_policy.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "key": { - "": { - "policy": "read" - }, - - "foo/": { - "policy": "write" - }, - - "foo/bar/": { - "policy": "read" - }, - - "foo/bar/baz": { - "policy": "deny" - } - } -} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.hcl deleted file mode 100644 index 52dcaa1b..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.hcl +++ /dev/null @@ -1,10 +0,0 @@ -variable "foo" { - default = "bar" - description = "bar" -} - -variable "amis" { - default = { - east = "foo" - } -} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.json b/vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.json deleted file mode 100644 index 49f921ed..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/decode_tf_variable.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "variable": { - "foo": { - "default": "bar", - "description": "bar" - }, - - "amis": { - "default": { - "east": "foo" - } - } - } -} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/empty.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/empty.hcl deleted file mode 100644 index 5be1b231..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/empty.hcl +++ /dev/null @@ -1 +0,0 @@ -resource "foo" {} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/escape.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/escape.hcl deleted file mode 100644 index ead1b8b9..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/escape.hcl +++ /dev/null @@ -1 +0,0 @@ -foo = "bar\"baz\\n" diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/flat.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/flat.hcl deleted file mode 100644 index 9bca551f..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/flat.hcl +++ /dev/null @@ -1,2 +0,0 @@ -foo = "bar" -Key = 7 diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/float.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/float.hcl deleted file mode 100644 index eed44e54..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/float.hcl +++ /dev/null @@ -1 +0,0 @@ -a = 1.02 diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/float.json b/vendor/github.com/hashicorp/hcl/test-fixtures/float.json deleted file mode 100644 index a9d1ab4b..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/float.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "a": 1.02 -} diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/interpolate_escape.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/interpolate_escape.hcl deleted file mode 100644 index 7b953913..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/interpolate_escape.hcl +++ /dev/null @@ -1 +0,0 @@ -foo="${file(\"bing/bong.txt\")}" diff --git a/vendor/github.com/hashicorp/hcl/test-fixtures/multiline.hcl b/vendor/github.com/hashicorp/hcl/test-fixtures/multiline.hcl deleted file mode 100644 index f883bd70..00000000 --- a/vendor/github.com/hashicorp/hcl/test-fixtures/multiline.hcl +++ /dev/null @@ -1,4 +0,0 @@ -foo = < float64(b.Max) { - return b.Max - } - //bump attempts count - b.attempts++ - //return as a time.Duration - return time.Duration(dur) + d := b.ForAttempt(b.attempt) + b.attempt++ + return d } -//Resets the current value of the counter back to Min -func (b *Backoff) Reset() { - b.attempts = 0 +const maxInt64 = float64(math.MaxInt64 - 512) + +// ForAttempt returns the duration for a specific attempt. This is useful if +// you have a large number of independent Backoffs, but don't want use +// unnecessary memory storing the Backoff parameters per Backoff. The first +// attempt should be 0. +// +// ForAttempt is concurrent-safe. +func (b *Backoff) ForAttempt(attempt float64) time.Duration { + // Zero-values are nonsensical, so we use + // them to apply defaults + min := b.Min + if min <= 0 { + min = 100 * time.Millisecond + } + max := b.Max + if max <= 0 { + max = 10 * time.Second + } + if min >= max { + // short-circuit + return max + } + factor := b.Factor + if factor <= 0 { + factor = 2 + } + //calculate this duration + minf := float64(min) + durf := minf * math.Pow(factor, attempt) + if b.Jitter { + durf = rand.Float64()*(durf-minf) + minf + } + //ensure float64 wont overflow int64 + if durf > maxInt64 { + return max + } + dur := time.Duration(durf) + //keep within bounds + if dur < min { + return min + } else if dur > max { + return max + } + return dur +} + +// Reset restarts the current attempt counter at zero. +func (b *Backoff) Reset() { + b.attempt = 0 +} + +// Attempt returns the current attempt counter value. +func (b *Backoff) Attempt() float64 { + return b.attempt } diff --git a/vendor/github.com/jpillora/backoff/backoff_test.go b/vendor/github.com/jpillora/backoff/backoff_test.go deleted file mode 100644 index c5feccef..00000000 --- a/vendor/github.com/jpillora/backoff/backoff_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package backoff - -import ( - "testing" - "time" -) - -func Test1(t *testing.T) { - - b := &Backoff{ - Min: 100 * time.Millisecond, - Max: 10 * time.Second, - Factor: 2, - } - - equals(t, b.Duration(), 100*time.Millisecond) - equals(t, b.Duration(), 200*time.Millisecond) - equals(t, b.Duration(), 400*time.Millisecond) - b.Reset() - equals(t, b.Duration(), 100*time.Millisecond) -} - -func Test2(t *testing.T) { - - b := &Backoff{ - Min: 100 * time.Millisecond, - Max: 10 * time.Second, - Factor: 1.5, - } - - equals(t, b.Duration(), 100*time.Millisecond) - equals(t, b.Duration(), 150*time.Millisecond) - equals(t, b.Duration(), 225*time.Millisecond) - b.Reset() - equals(t, b.Duration(), 100*time.Millisecond) -} - -func Test3(t *testing.T) { - - b := &Backoff{ - Min: 100 * time.Nanosecond, - Max: 10 * time.Second, - Factor: 1.75, - } - - equals(t, b.Duration(), 100*time.Nanosecond) - equals(t, b.Duration(), 175*time.Nanosecond) - equals(t, b.Duration(), 306*time.Nanosecond) - b.Reset() - equals(t, b.Duration(), 100*time.Nanosecond) -} - -func TestJitter(t *testing.T) { - b := &Backoff{ - Min: 100 * time.Millisecond, - Max: 10 * time.Second, - Factor: 2, - Jitter: true, - } - - equals(t, b.Duration(), 100*time.Millisecond) - between(t, b.Duration(), 100*time.Millisecond, 200*time.Millisecond) - between(t, b.Duration(), 100*time.Millisecond, 400*time.Millisecond) - b.Reset() - equals(t, b.Duration(), 100*time.Millisecond) -} - -func between(t *testing.T, actual, low, high time.Duration) { - if actual < low { - t.Fatalf("Got %s, Expecting >= %s", actual, low) - } - if actual > high { - t.Fatalf("Got %s, Expecting <= %s", actual, high) - } -} - -func equals(t *testing.T, d1, d2 time.Duration) { - if d1 != d2 { - t.Fatalf("Got %s, Expecting %s", d1, d2) - } -} diff --git a/vendor/github.com/kr/pretty/Readme b/vendor/github.com/kr/pretty/Readme deleted file mode 100644 index c589fc62..00000000 --- a/vendor/github.com/kr/pretty/Readme +++ /dev/null @@ -1,9 +0,0 @@ -package pretty - - import "github.com/kr/pretty" - - Package pretty provides pretty-printing for Go values. - -Documentation - - http://godoc.org/github.com/kr/pretty diff --git a/vendor/github.com/kr/pretty/diff.go b/vendor/github.com/kr/pretty/diff.go deleted file mode 100644 index 8fe8e240..00000000 --- a/vendor/github.com/kr/pretty/diff.go +++ /dev/null @@ -1,158 +0,0 @@ -package pretty - -import ( - "fmt" - "io" - "reflect" -) - -type sbuf []string - -func (s *sbuf) Write(b []byte) (int, error) { - *s = append(*s, string(b)) - return len(b), nil -} - -// Diff returns a slice where each element describes -// a difference between a and b. -func Diff(a, b interface{}) (desc []string) { - Fdiff((*sbuf)(&desc), a, b) - return desc -} - -// Fdiff writes to w a description of the differences between a and b. -func Fdiff(w io.Writer, a, b interface{}) { - diffWriter{w: w}.diff(reflect.ValueOf(a), reflect.ValueOf(b)) -} - -type diffWriter struct { - w io.Writer - l string // label -} - -func (w diffWriter) printf(f string, a ...interface{}) { - var l string - if w.l != "" { - l = w.l + ": " - } - fmt.Fprintf(w.w, l+f, a...) -} - -func (w diffWriter) diff(av, bv reflect.Value) { - if !av.IsValid() && bv.IsValid() { - w.printf("nil != %#v", bv.Interface()) - return - } - if av.IsValid() && !bv.IsValid() { - w.printf("%#v != nil", av.Interface()) - return - } - if !av.IsValid() && !bv.IsValid() { - return - } - - at := av.Type() - bt := bv.Type() - if at != bt { - w.printf("%v != %v", at, bt) - return - } - - // numeric types, including bool - if at.Kind() < reflect.Array { - a, b := av.Interface(), bv.Interface() - if a != b { - w.printf("%#v != %#v", a, b) - } - return - } - - switch at.Kind() { - case reflect.String: - a, b := av.Interface(), bv.Interface() - if a != b { - w.printf("%q != %q", a, b) - } - case reflect.Ptr: - switch { - case av.IsNil() && !bv.IsNil(): - w.printf("nil != %v", bv.Interface()) - case !av.IsNil() && bv.IsNil(): - w.printf("%v != nil", av.Interface()) - case !av.IsNil() && !bv.IsNil(): - w.diff(av.Elem(), bv.Elem()) - } - case reflect.Struct: - for i := 0; i < av.NumField(); i++ { - w.relabel(at.Field(i).Name).diff(av.Field(i), bv.Field(i)) - } - case reflect.Slice: - lenA := av.Len() - lenB := bv.Len() - if lenA != lenB { - w.printf("%s[%d] != %s[%d]", av.Type(), lenA, bv.Type(), lenB) - break - } - for i := 0; i < lenA; i++ { - w.relabel(fmt.Sprintf("[%d]", i)).diff(av.Index(i), bv.Index(i)) - } - case reflect.Map: - ak, both, bk := keyDiff(av.MapKeys(), bv.MapKeys()) - for _, k := range ak { - w := w.relabel(fmt.Sprintf("[%#v]", k.Interface())) - w.printf("%q != (missing)", av.MapIndex(k)) - } - for _, k := range both { - w := w.relabel(fmt.Sprintf("[%#v]", k.Interface())) - w.diff(av.MapIndex(k), bv.MapIndex(k)) - } - for _, k := range bk { - w := w.relabel(fmt.Sprintf("[%#v]", k.Interface())) - w.printf("(missing) != %q", bv.MapIndex(k)) - } - case reflect.Interface: - w.diff(reflect.ValueOf(av.Interface()), reflect.ValueOf(bv.Interface())) - default: - if !reflect.DeepEqual(av.Interface(), bv.Interface()) { - w.printf("%# v != %# v", Formatter(av.Interface()), Formatter(bv.Interface())) - } - } -} - -func (d diffWriter) relabel(name string) (d1 diffWriter) { - d1 = d - if d.l != "" && name[0] != '[' { - d1.l += "." - } - d1.l += name - return d1 -} - -func keyDiff(a, b []reflect.Value) (ak, both, bk []reflect.Value) { - for _, av := range a { - inBoth := false - for _, bv := range b { - if reflect.DeepEqual(av.Interface(), bv.Interface()) { - inBoth = true - both = append(both, av) - break - } - } - if !inBoth { - ak = append(ak, av) - } - } - for _, bv := range b { - inBoth := false - for _, av := range a { - if reflect.DeepEqual(av.Interface(), bv.Interface()) { - inBoth = true - break - } - } - if !inBoth { - bk = append(bk, bv) - } - } - return -} diff --git a/vendor/github.com/kr/pretty/diff_test.go b/vendor/github.com/kr/pretty/diff_test.go deleted file mode 100644 index 3c388f13..00000000 --- a/vendor/github.com/kr/pretty/diff_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package pretty - -import ( - "testing" -) - -type difftest struct { - a interface{} - b interface{} - exp []string -} - -type S struct { - A int - S *S - I interface{} - C []int -} - -var diffs = []difftest{ - {a: nil, b: nil}, - {a: S{A: 1}, b: S{A: 1}}, - - {0, "", []string{`int != string`}}, - {0, 1, []string{`0 != 1`}}, - {S{}, new(S), []string{`pretty.S != *pretty.S`}}, - {"a", "b", []string{`"a" != "b"`}}, - {S{}, S{A: 1}, []string{`A: 0 != 1`}}, - {new(S), &S{A: 1}, []string{`A: 0 != 1`}}, - {S{S: new(S)}, S{S: &S{A: 1}}, []string{`S.A: 0 != 1`}}, - {S{}, S{I: 0}, []string{`I: nil != 0`}}, - {S{I: 1}, S{I: "x"}, []string{`I: int != string`}}, - {S{}, S{C: []int{1}}, []string{`C: []int[0] != []int[1]`}}, - {S{C: []int{}}, S{C: []int{1}}, []string{`C: []int[0] != []int[1]`}}, - {S{C: []int{1, 2, 3}}, S{C: []int{1, 2, 4}}, []string{`C[2]: 3 != 4`}}, - {S{}, S{A: 1, S: new(S)}, []string{`A: 0 != 1`, `S: nil != &{0 []}`}}, -} - -func TestDiff(t *testing.T) { - for _, tt := range diffs { - got := Diff(tt.a, tt.b) - eq := len(got) == len(tt.exp) - if eq { - for i := range got { - eq = eq && got[i] == tt.exp[i] - } - } - if !eq { - t.Errorf("diffing % #v", tt.a) - t.Errorf("with % #v", tt.b) - diffdiff(t, got, tt.exp) - continue - } - } -} - -func diffdiff(t *testing.T, got, exp []string) { - minus(t, "unexpected:", got, exp) - minus(t, "missing:", exp, got) -} - -func minus(t *testing.T, s string, a, b []string) { - var i, j int - for i = 0; i < len(a); i++ { - for j = 0; j < len(b); j++ { - if a[i] == b[j] { - break - } - } - if j == len(b) { - t.Error(s, a[i]) - } - } -} diff --git a/vendor/github.com/kr/pretty/example_test.go b/vendor/github.com/kr/pretty/example_test.go deleted file mode 100644 index ecf40f3f..00000000 --- a/vendor/github.com/kr/pretty/example_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package pretty_test - -import ( - "fmt" - "github.com/kr/pretty" -) - -func Example() { - type myType struct { - a, b int - } - var x = []myType{{1, 2}, {3, 4}, {5, 6}} - fmt.Printf("%# v", pretty.Formatter(x)) - // output: - // []pretty_test.myType{ - // {a:1, b:2}, - // {a:3, b:4}, - // {a:5, b:6}, - // } -} diff --git a/vendor/github.com/kr/pretty/formatter.go b/vendor/github.com/kr/pretty/formatter.go deleted file mode 100644 index 8dacda25..00000000 --- a/vendor/github.com/kr/pretty/formatter.go +++ /dev/null @@ -1,337 +0,0 @@ -package pretty - -import ( - "fmt" - "io" - "reflect" - "strconv" - "text/tabwriter" - - "github.com/kr/text" -) - -const ( - limit = 50 -) - -type formatter struct { - x interface{} - force bool - quote bool -} - -// Formatter makes a wrapper, f, that will format x as go source with line -// breaks and tabs. Object f responds to the "%v" formatting verb when both the -// "#" and " " (space) flags are set, for example: -// -// fmt.Sprintf("%# v", Formatter(x)) -// -// If one of these two flags is not set, or any other verb is used, f will -// format x according to the usual rules of package fmt. -// In particular, if x satisfies fmt.Formatter, then x.Format will be called. -func Formatter(x interface{}) (f fmt.Formatter) { - return formatter{x: x, quote: true} -} - -func (fo formatter) String() string { - return fmt.Sprint(fo.x) // unwrap it -} - -func (fo formatter) passThrough(f fmt.State, c rune) { - s := "%" - for i := 0; i < 128; i++ { - if f.Flag(i) { - s += string(i) - } - } - if w, ok := f.Width(); ok { - s += fmt.Sprintf("%d", w) - } - if p, ok := f.Precision(); ok { - s += fmt.Sprintf(".%d", p) - } - s += string(c) - fmt.Fprintf(f, s, fo.x) -} - -func (fo formatter) Format(f fmt.State, c rune) { - if fo.force || c == 'v' && f.Flag('#') && f.Flag(' ') { - w := tabwriter.NewWriter(f, 4, 4, 1, ' ', 0) - p := &printer{tw: w, Writer: w, visited: make(map[visit]int)} - p.printValue(reflect.ValueOf(fo.x), true, fo.quote) - w.Flush() - return - } - fo.passThrough(f, c) -} - -type printer struct { - io.Writer - tw *tabwriter.Writer - visited map[visit]int - depth int -} - -func (p *printer) indent() *printer { - q := *p - q.tw = tabwriter.NewWriter(p.Writer, 4, 4, 1, ' ', 0) - q.Writer = text.NewIndentWriter(q.tw, []byte{'\t'}) - return &q -} - -func (p *printer) printInline(v reflect.Value, x interface{}, showType bool) { - if showType { - io.WriteString(p, v.Type().String()) - fmt.Fprintf(p, "(%#v)", x) - } else { - fmt.Fprintf(p, "%#v", x) - } -} - -// printValue must keep track of already-printed pointer values to avoid -// infinite recursion. -type visit struct { - v uintptr - typ reflect.Type -} - -func (p *printer) printValue(v reflect.Value, showType, quote bool) { - if p.depth > 10 { - io.WriteString(p, "!%v(DEPTH EXCEEDED)") - return - } - - switch v.Kind() { - case reflect.Bool: - p.printInline(v, v.Bool(), showType) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - p.printInline(v, v.Int(), showType) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - p.printInline(v, v.Uint(), showType) - case reflect.Float32, reflect.Float64: - p.printInline(v, v.Float(), showType) - case reflect.Complex64, reflect.Complex128: - fmt.Fprintf(p, "%#v", v.Complex()) - case reflect.String: - p.fmtString(v.String(), quote) - case reflect.Map: - t := v.Type() - if showType { - io.WriteString(p, t.String()) - } - writeByte(p, '{') - if nonzero(v) { - expand := !canInline(v.Type()) - pp := p - if expand { - writeByte(p, '\n') - pp = p.indent() - } - keys := v.MapKeys() - for i := 0; i < v.Len(); i++ { - showTypeInStruct := true - k := keys[i] - mv := v.MapIndex(k) - pp.printValue(k, false, true) - writeByte(pp, ':') - if expand { - writeByte(pp, '\t') - } - showTypeInStruct = t.Elem().Kind() == reflect.Interface - pp.printValue(mv, showTypeInStruct, true) - if expand { - io.WriteString(pp, ",\n") - } else if i < v.Len()-1 { - io.WriteString(pp, ", ") - } - } - if expand { - pp.tw.Flush() - } - } - writeByte(p, '}') - case reflect.Struct: - t := v.Type() - if v.CanAddr() { - addr := v.UnsafeAddr() - vis := visit{addr, t} - if vd, ok := p.visited[vis]; ok && vd < p.depth { - p.fmtString(t.String()+"{(CYCLIC REFERENCE)}", false) - break // don't print v again - } - p.visited[vis] = p.depth - } - - if showType { - io.WriteString(p, t.String()) - } - writeByte(p, '{') - if nonzero(v) { - expand := !canInline(v.Type()) - pp := p - if expand { - writeByte(p, '\n') - pp = p.indent() - } - for i := 0; i < v.NumField(); i++ { - showTypeInStruct := true - if f := t.Field(i); f.Name != "" { - io.WriteString(pp, f.Name) - writeByte(pp, ':') - if expand { - writeByte(pp, '\t') - } - showTypeInStruct = labelType(f.Type) - } - pp.printValue(getField(v, i), showTypeInStruct, true) - if expand { - io.WriteString(pp, ",\n") - } else if i < v.NumField()-1 { - io.WriteString(pp, ", ") - } - } - if expand { - pp.tw.Flush() - } - } - writeByte(p, '}') - case reflect.Interface: - switch e := v.Elem(); { - case e.Kind() == reflect.Invalid: - io.WriteString(p, "nil") - case e.IsValid(): - pp := *p - pp.depth++ - pp.printValue(e, showType, true) - default: - io.WriteString(p, v.Type().String()) - io.WriteString(p, "(nil)") - } - case reflect.Array, reflect.Slice: - t := v.Type() - if showType { - io.WriteString(p, t.String()) - } - if v.Kind() == reflect.Slice && v.IsNil() && showType { - io.WriteString(p, "(nil)") - break - } - if v.Kind() == reflect.Slice && v.IsNil() { - io.WriteString(p, "nil") - break - } - writeByte(p, '{') - expand := !canInline(v.Type()) - pp := p - if expand { - writeByte(p, '\n') - pp = p.indent() - } - for i := 0; i < v.Len(); i++ { - showTypeInSlice := t.Elem().Kind() == reflect.Interface - pp.printValue(v.Index(i), showTypeInSlice, true) - if expand { - io.WriteString(pp, ",\n") - } else if i < v.Len()-1 { - io.WriteString(pp, ", ") - } - } - if expand { - pp.tw.Flush() - } - writeByte(p, '}') - case reflect.Ptr: - e := v.Elem() - if !e.IsValid() { - writeByte(p, '(') - io.WriteString(p, v.Type().String()) - io.WriteString(p, ")(nil)") - } else { - pp := *p - pp.depth++ - writeByte(pp, '&') - pp.printValue(e, true, true) - } - case reflect.Chan: - x := v.Pointer() - if showType { - writeByte(p, '(') - io.WriteString(p, v.Type().String()) - fmt.Fprintf(p, ")(%#v)", x) - } else { - fmt.Fprintf(p, "%#v", x) - } - case reflect.Func: - io.WriteString(p, v.Type().String()) - io.WriteString(p, " {...}") - case reflect.UnsafePointer: - p.printInline(v, v.Pointer(), showType) - case reflect.Invalid: - io.WriteString(p, "nil") - } -} - -func canInline(t reflect.Type) bool { - switch t.Kind() { - case reflect.Map: - return !canExpand(t.Elem()) - case reflect.Struct: - for i := 0; i < t.NumField(); i++ { - if canExpand(t.Field(i).Type) { - return false - } - } - return true - case reflect.Interface: - return false - case reflect.Array, reflect.Slice: - return !canExpand(t.Elem()) - case reflect.Ptr: - return false - case reflect.Chan, reflect.Func, reflect.UnsafePointer: - return false - } - return true -} - -func canExpand(t reflect.Type) bool { - switch t.Kind() { - case reflect.Map, reflect.Struct, - reflect.Interface, reflect.Array, reflect.Slice, - reflect.Ptr: - return true - } - return false -} - -func labelType(t reflect.Type) bool { - switch t.Kind() { - case reflect.Interface, reflect.Struct: - return true - } - return false -} - -func (p *printer) fmtString(s string, quote bool) { - if quote { - s = strconv.Quote(s) - } - io.WriteString(p, s) -} - -func tryDeepEqual(a, b interface{}) bool { - defer func() { recover() }() - return reflect.DeepEqual(a, b) -} - -func writeByte(w io.Writer, b byte) { - w.Write([]byte{b}) -} - -func getField(v reflect.Value, i int) reflect.Value { - val := v.Field(i) - if val.Kind() == reflect.Interface && !val.IsNil() { - val = val.Elem() - } - return val -} diff --git a/vendor/github.com/kr/pretty/formatter_test.go b/vendor/github.com/kr/pretty/formatter_test.go deleted file mode 100644 index 5f3204e8..00000000 --- a/vendor/github.com/kr/pretty/formatter_test.go +++ /dev/null @@ -1,261 +0,0 @@ -package pretty - -import ( - "fmt" - "io" - "strings" - "testing" - "unsafe" -) - -type test struct { - v interface{} - s string -} - -type LongStructTypeName struct { - longFieldName interface{} - otherLongFieldName interface{} -} - -type SA struct { - t *T - v T -} - -type T struct { - x, y int -} - -type F int - -func (f F) Format(s fmt.State, c rune) { - fmt.Fprintf(s, "F(%d)", int(f)) -} - -var long = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - -var gosyntax = []test{ - {nil, `nil`}, - {"", `""`}, - {"a", `"a"`}, - {1, "int(1)"}, - {1.0, "float64(1)"}, - {[]int(nil), "[]int(nil)"}, - {[0]int{}, "[0]int{}"}, - {complex(1, 0), "(1+0i)"}, - //{make(chan int), "(chan int)(0x1234)"}, - {unsafe.Pointer(uintptr(unsafe.Pointer(&long))), fmt.Sprintf("unsafe.Pointer(0x%02x)", uintptr(unsafe.Pointer(&long)))}, - {func(int) {}, "func(int) {...}"}, - {map[int]int{1: 1}, "map[int]int{1:1}"}, - {int32(1), "int32(1)"}, - {io.EOF, `&errors.errorString{s:"EOF"}`}, - {[]string{"a"}, `[]string{"a"}`}, - { - []string{long}, - `[]string{"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"}`, - }, - {F(5), "pretty.F(5)"}, - { - SA{&T{1, 2}, T{3, 4}}, - `pretty.SA{ - t: &pretty.T{x:1, y:2}, - v: pretty.T{x:3, y:4}, -}`, - }, - { - map[int][]byte{1: {}}, - `map[int][]uint8{ - 1: {}, -}`, - }, - { - map[int]T{1: {}}, - `map[int]pretty.T{ - 1: {}, -}`, - }, - { - long, - `"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"`, - }, - { - LongStructTypeName{ - longFieldName: LongStructTypeName{}, - otherLongFieldName: long, - }, - `pretty.LongStructTypeName{ - longFieldName: pretty.LongStructTypeName{}, - otherLongFieldName: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", -}`, - }, - { - &LongStructTypeName{ - longFieldName: &LongStructTypeName{}, - otherLongFieldName: (*LongStructTypeName)(nil), - }, - `&pretty.LongStructTypeName{ - longFieldName: &pretty.LongStructTypeName{}, - otherLongFieldName: (*pretty.LongStructTypeName)(nil), -}`, - }, - { - []LongStructTypeName{ - {nil, nil}, - {3, 3}, - {long, nil}, - }, - `[]pretty.LongStructTypeName{ - {}, - { - longFieldName: int(3), - otherLongFieldName: int(3), - }, - { - longFieldName: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", - otherLongFieldName: nil, - }, -}`, - }, - { - []interface{}{ - LongStructTypeName{nil, nil}, - []byte{1, 2, 3}, - T{3, 4}, - LongStructTypeName{long, nil}, - }, - `[]interface {}{ - pretty.LongStructTypeName{}, - []uint8{0x1, 0x2, 0x3}, - pretty.T{x:3, y:4}, - pretty.LongStructTypeName{ - longFieldName: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", - otherLongFieldName: nil, - }, -}`, - }, -} - -func TestGoSyntax(t *testing.T) { - for _, tt := range gosyntax { - s := fmt.Sprintf("%# v", Formatter(tt.v)) - if tt.s != s { - t.Errorf("expected %q", tt.s) - t.Errorf("got %q", s) - t.Errorf("expraw\n%s", tt.s) - t.Errorf("gotraw\n%s", s) - } - } -} - -type I struct { - i int - R interface{} -} - -func (i *I) I() *I { return i.R.(*I) } - -func TestCycle(t *testing.T) { - type A struct{ *A } - v := &A{} - v.A = v - - // panics from stack overflow without cycle detection - t.Logf("Example cycle:\n%# v", Formatter(v)) - - p := &A{} - s := fmt.Sprintf("%# v", Formatter([]*A{p, p})) - if strings.Contains(s, "CYCLIC") { - t.Errorf("Repeated address detected as cyclic reference:\n%s", s) - } - - type R struct { - i int - *R - } - r := &R{ - i: 1, - R: &R{ - i: 2, - R: &R{ - i: 3, - }, - }, - } - r.R.R.R = r - t.Logf("Example longer cycle:\n%# v", Formatter(r)) - - r = &R{ - i: 1, - R: &R{ - i: 2, - R: &R{ - i: 3, - R: &R{ - i: 4, - R: &R{ - i: 5, - R: &R{ - i: 6, - R: &R{ - i: 7, - R: &R{ - i: 8, - R: &R{ - i: 9, - R: &R{ - i: 10, - R: &R{ - i: 11, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - // here be pirates - r.R.R.R.R.R.R.R.R.R.R.R = r - t.Logf("Example very long cycle:\n%# v", Formatter(r)) - - i := &I{ - i: 1, - R: &I{ - i: 2, - R: &I{ - i: 3, - R: &I{ - i: 4, - R: &I{ - i: 5, - R: &I{ - i: 6, - R: &I{ - i: 7, - R: &I{ - i: 8, - R: &I{ - i: 9, - R: &I{ - i: 10, - R: &I{ - i: 11, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - iv := i.I().I().I().I().I().I().I().I().I().I() - *iv = *i - t.Logf("Example long interface cycle:\n%# v", Formatter(i)) -} diff --git a/vendor/github.com/kr/pretty/pretty.go b/vendor/github.com/kr/pretty/pretty.go deleted file mode 100644 index d3df8686..00000000 --- a/vendor/github.com/kr/pretty/pretty.go +++ /dev/null @@ -1,98 +0,0 @@ -// Package pretty provides pretty-printing for Go values. This is -// useful during debugging, to avoid wrapping long output lines in -// the terminal. -// -// It provides a function, Formatter, that can be used with any -// function that accepts a format string. It also provides -// convenience wrappers for functions in packages fmt and log. -package pretty - -import ( - "fmt" - "io" - "log" -) - -// Errorf is a convenience wrapper for fmt.Errorf. -// -// Calling Errorf(f, x, y) is equivalent to -// fmt.Errorf(f, Formatter(x), Formatter(y)). -func Errorf(format string, a ...interface{}) error { - return fmt.Errorf(format, wrap(a, false)...) -} - -// Fprintf is a convenience wrapper for fmt.Fprintf. -// -// Calling Fprintf(w, f, x, y) is equivalent to -// fmt.Fprintf(w, f, Formatter(x), Formatter(y)). -func Fprintf(w io.Writer, format string, a ...interface{}) (n int, error error) { - return fmt.Fprintf(w, format, wrap(a, false)...) -} - -// Log is a convenience wrapper for log.Printf. -// -// Calling Log(x, y) is equivalent to -// log.Print(Formatter(x), Formatter(y)), but each operand is -// formatted with "%# v". -func Log(a ...interface{}) { - log.Print(wrap(a, true)...) -} - -// Logf is a convenience wrapper for log.Printf. -// -// Calling Logf(f, x, y) is equivalent to -// log.Printf(f, Formatter(x), Formatter(y)). -func Logf(format string, a ...interface{}) { - log.Printf(format, wrap(a, false)...) -} - -// Logln is a convenience wrapper for log.Printf. -// -// Calling Logln(x, y) is equivalent to -// log.Println(Formatter(x), Formatter(y)), but each operand is -// formatted with "%# v". -func Logln(a ...interface{}) { - log.Println(wrap(a, true)...) -} - -// Print pretty-prints its operands and writes to standard output. -// -// Calling Print(x, y) is equivalent to -// fmt.Print(Formatter(x), Formatter(y)), but each operand is -// formatted with "%# v". -func Print(a ...interface{}) (n int, errno error) { - return fmt.Print(wrap(a, true)...) -} - -// Printf is a convenience wrapper for fmt.Printf. -// -// Calling Printf(f, x, y) is equivalent to -// fmt.Printf(f, Formatter(x), Formatter(y)). -func Printf(format string, a ...interface{}) (n int, errno error) { - return fmt.Printf(format, wrap(a, false)...) -} - -// Println pretty-prints its operands and writes to standard output. -// -// Calling Print(x, y) is equivalent to -// fmt.Println(Formatter(x), Formatter(y)), but each operand is -// formatted with "%# v". -func Println(a ...interface{}) (n int, errno error) { - return fmt.Println(wrap(a, true)...) -} - -// Sprintf is a convenience wrapper for fmt.Sprintf. -// -// Calling Sprintf(f, x, y) is equivalent to -// fmt.Sprintf(f, Formatter(x), Formatter(y)). -func Sprintf(format string, a ...interface{}) string { - return fmt.Sprintf(format, wrap(a, false)...) -} - -func wrap(a []interface{}, force bool) []interface{} { - w := make([]interface{}, len(a)) - for i, x := range a { - w[i] = formatter{x: x, force: force} - } - return w -} diff --git a/vendor/github.com/kr/pretty/zero.go b/vendor/github.com/kr/pretty/zero.go deleted file mode 100644 index abb5b6fc..00000000 --- a/vendor/github.com/kr/pretty/zero.go +++ /dev/null @@ -1,41 +0,0 @@ -package pretty - -import ( - "reflect" -) - -func nonzero(v reflect.Value) bool { - switch v.Kind() { - case reflect.Bool: - return v.Bool() - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return v.Int() != 0 - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return v.Uint() != 0 - case reflect.Float32, reflect.Float64: - return v.Float() != 0 - case reflect.Complex64, reflect.Complex128: - return v.Complex() != complex(0, 0) - case reflect.String: - return v.String() != "" - case reflect.Struct: - for i := 0; i < v.NumField(); i++ { - if nonzero(getField(v, i)) { - return true - } - } - return false - case reflect.Array: - for i := 0; i < v.Len(); i++ { - if nonzero(v.Index(i)) { - return true - } - } - return false - case reflect.Map, reflect.Interface, reflect.Slice, reflect.Ptr, reflect.Chan, reflect.Func: - return !v.IsNil() - case reflect.UnsafePointer: - return v.Pointer() != 0 - } - return true -} diff --git a/vendor/github.com/kr/text/Readme b/vendor/github.com/kr/text/Readme deleted file mode 100644 index 7e6e7c06..00000000 --- a/vendor/github.com/kr/text/Readme +++ /dev/null @@ -1,3 +0,0 @@ -This is a Go package for manipulating paragraphs of text. - -See http://go.pkgdoc.org/github.com/kr/text for full documentation. diff --git a/vendor/github.com/kr/text/colwriter/Readme b/vendor/github.com/kr/text/colwriter/Readme deleted file mode 100644 index 1c1f4e68..00000000 --- a/vendor/github.com/kr/text/colwriter/Readme +++ /dev/null @@ -1,5 +0,0 @@ -Package colwriter provides a write filter that formats -input lines in multiple columns. - -The package is a straightforward translation from -/src/cmd/draw/mc.c in Plan 9 from User Space. diff --git a/vendor/github.com/kr/text/colwriter/column.go b/vendor/github.com/kr/text/colwriter/column.go deleted file mode 100644 index 7302ce9f..00000000 --- a/vendor/github.com/kr/text/colwriter/column.go +++ /dev/null @@ -1,147 +0,0 @@ -// Package colwriter provides a write filter that formats -// input lines in multiple columns. -// -// The package is a straightforward translation from -// /src/cmd/draw/mc.c in Plan 9 from User Space. -package colwriter - -import ( - "bytes" - "io" - "unicode/utf8" -) - -const ( - tab = 4 -) - -const ( - // Print each input line ending in a colon ':' separately. - BreakOnColon uint = 1 << iota -) - -// A Writer is a filter that arranges input lines in as many columns as will -// fit in its width. Tab '\t' chars in the input are translated to sequences -// of spaces ending at multiples of 4 positions. -// -// If BreakOnColon is set, each input line ending in a colon ':' is written -// separately. -// -// The Writer assumes that all Unicode code points have the same width; this -// may not be true in some fonts. -type Writer struct { - w io.Writer - buf []byte - width int - flag uint -} - -// NewWriter allocates and initializes a new Writer writing to w. -// Parameter width controls the total number of characters on each line -// across all columns. -func NewWriter(w io.Writer, width int, flag uint) *Writer { - return &Writer{ - w: w, - width: width, - flag: flag, - } -} - -// Write writes p to the writer w. The only errors returned are ones -// encountered while writing to the underlying output stream. -func (w *Writer) Write(p []byte) (n int, err error) { - var linelen int - var lastWasColon bool - for i, c := range p { - w.buf = append(w.buf, c) - linelen++ - if c == '\t' { - w.buf[len(w.buf)-1] = ' ' - for linelen%tab != 0 { - w.buf = append(w.buf, ' ') - linelen++ - } - } - if w.flag&BreakOnColon != 0 && c == ':' { - lastWasColon = true - } else if lastWasColon { - if c == '\n' { - pos := bytes.LastIndex(w.buf[:len(w.buf)-1], []byte{'\n'}) - if pos < 0 { - pos = 0 - } - line := w.buf[pos:] - w.buf = w.buf[:pos] - if err = w.columnate(); err != nil { - if len(line) < i { - return i - len(line), err - } - return 0, err - } - if n, err := w.w.Write(line); err != nil { - if r := len(line) - n; r < i { - return i - r, err - } - return 0, err - } - } - lastWasColon = false - } - if c == '\n' { - linelen = 0 - } - } - return len(p), nil -} - -// Flush should be called after the last call to Write to ensure that any data -// buffered in the Writer is written to output. -func (w *Writer) Flush() error { - return w.columnate() -} - -func (w *Writer) columnate() error { - words := bytes.Split(w.buf, []byte{'\n'}) - w.buf = nil - if len(words[len(words)-1]) == 0 { - words = words[:len(words)-1] - } - maxwidth := 0 - for _, wd := range words { - if n := utf8.RuneCount(wd); n > maxwidth { - maxwidth = n - } - } - maxwidth++ // space char - wordsPerLine := w.width / maxwidth - if wordsPerLine <= 0 { - wordsPerLine = 1 - } - nlines := (len(words) + wordsPerLine - 1) / wordsPerLine - for i := 0; i < nlines; i++ { - col := 0 - endcol := 0 - for j := i; j < len(words); j += nlines { - endcol += maxwidth - _, err := w.w.Write(words[j]) - if err != nil { - return err - } - col += utf8.RuneCount(words[j]) - if j+nlines < len(words) { - for col < endcol { - _, err := w.w.Write([]byte{' '}) - if err != nil { - return err - } - col++ - } - } - } - _, err := w.w.Write([]byte{'\n'}) - if err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/kr/text/colwriter/column_test.go b/vendor/github.com/kr/text/colwriter/column_test.go deleted file mode 100644 index ce388f5a..00000000 --- a/vendor/github.com/kr/text/colwriter/column_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package colwriter - -import ( - "bytes" - "testing" -) - -var src = ` -.git -.gitignore -.godir -Procfile: -README.md -api.go -apps.go -auth.go -darwin.go -data.go -dyno.go: -env.go -git.go -help.go -hkdist -linux.go -ls.go -main.go -plugin.go -run.go -scale.go -ssh.go -tail.go -term -unix.go -update.go -version.go -windows.go -`[1:] - -var tests = []struct { - wid int - flag uint - src string - want string -}{ - {80, 0, "", ""}, - {80, 0, src, ` -.git README.md darwin.go git.go ls.go scale.go unix.go -.gitignore api.go data.go help.go main.go ssh.go update.go -.godir apps.go dyno.go: hkdist plugin.go tail.go version.go -Procfile: auth.go env.go linux.go run.go term windows.go -`[1:]}, - {80, BreakOnColon, src, ` -.git .gitignore .godir - -Procfile: -README.md api.go apps.go auth.go darwin.go data.go - -dyno.go: -env.go hkdist main.go scale.go term version.go -git.go linux.go plugin.go ssh.go unix.go windows.go -help.go ls.go run.go tail.go update.go -`[1:]}, - {20, 0, ` -Hello -Γειά σου -안녕 -今日は -`[1:], ` -Hello 안녕 -Γειά σου 今日は -`[1:]}, -} - -func TestWriter(t *testing.T) { - for _, test := range tests { - b := new(bytes.Buffer) - w := NewWriter(b, test.wid, test.flag) - if _, err := w.Write([]byte(test.src)); err != nil { - t.Error(err) - } - if err := w.Flush(); err != nil { - t.Error(err) - } - if g := b.String(); test.want != g { - t.Log("\n" + test.want) - t.Log("\n" + g) - t.Errorf("%q != %q", test.want, g) - } - } -} diff --git a/vendor/github.com/kr/text/doc.go b/vendor/github.com/kr/text/doc.go deleted file mode 100644 index cf4c198f..00000000 --- a/vendor/github.com/kr/text/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package text provides rudimentary functions for manipulating text in -// paragraphs. -package text diff --git a/vendor/github.com/kr/text/indent.go b/vendor/github.com/kr/text/indent.go deleted file mode 100644 index 4ebac45c..00000000 --- a/vendor/github.com/kr/text/indent.go +++ /dev/null @@ -1,74 +0,0 @@ -package text - -import ( - "io" -) - -// Indent inserts prefix at the beginning of each non-empty line of s. The -// end-of-line marker is NL. -func Indent(s, prefix string) string { - return string(IndentBytes([]byte(s), []byte(prefix))) -} - -// IndentBytes inserts prefix at the beginning of each non-empty line of b. -// The end-of-line marker is NL. -func IndentBytes(b, prefix []byte) []byte { - var res []byte - bol := true - for _, c := range b { - if bol && c != '\n' { - res = append(res, prefix...) - } - res = append(res, c) - bol = c == '\n' - } - return res -} - -// Writer indents each line of its input. -type indentWriter struct { - w io.Writer - bol bool - pre [][]byte - sel int - off int -} - -// NewIndentWriter makes a new write filter that indents the input -// lines. Each line is prefixed in order with the corresponding -// element of pre. If there are more lines than elements, the last -// element of pre is repeated for each subsequent line. -func NewIndentWriter(w io.Writer, pre ...[]byte) io.Writer { - return &indentWriter{ - w: w, - pre: pre, - bol: true, - } -} - -// The only errors returned are from the underlying indentWriter. -func (w *indentWriter) Write(p []byte) (n int, err error) { - for _, c := range p { - if w.bol { - var i int - i, err = w.w.Write(w.pre[w.sel][w.off:]) - w.off += i - if err != nil { - return n, err - } - } - _, err = w.w.Write([]byte{c}) - if err != nil { - return n, err - } - n++ - w.bol = c == '\n' - if w.bol { - w.off = 0 - if w.sel < len(w.pre)-1 { - w.sel++ - } - } - } - return n, nil -} diff --git a/vendor/github.com/kr/text/indent_test.go b/vendor/github.com/kr/text/indent_test.go deleted file mode 100644 index 5c723eee..00000000 --- a/vendor/github.com/kr/text/indent_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package text - -import ( - "bytes" - "testing" -) - -type T struct { - inp, exp, pre string -} - -var tests = []T{ - { - "The quick brown fox\njumps over the lazy\ndog.\nBut not quickly.\n", - "xxxThe quick brown fox\nxxxjumps over the lazy\nxxxdog.\nxxxBut not quickly.\n", - "xxx", - }, - { - "The quick brown fox\njumps over the lazy\ndog.\n\nBut not quickly.", - "xxxThe quick brown fox\nxxxjumps over the lazy\nxxxdog.\n\nxxxBut not quickly.", - "xxx", - }, -} - -func TestIndent(t *testing.T) { - for _, test := range tests { - got := Indent(test.inp, test.pre) - if got != test.exp { - t.Errorf("mismatch %q != %q", got, test.exp) - } - } -} - -type IndentWriterTest struct { - inp, exp string - pre []string -} - -var ts = []IndentWriterTest{ - { - ` -The quick brown fox -jumps over the lazy -dog. -But not quickly. -`[1:], - ` -xxxThe quick brown fox -xxxjumps over the lazy -xxxdog. -xxxBut not quickly. -`[1:], - []string{"xxx"}, - }, - { - ` -The quick brown fox -jumps over the lazy -dog. -But not quickly. -`[1:], - ` -xxaThe quick brown fox -xxxjumps over the lazy -xxxdog. -xxxBut not quickly. -`[1:], - []string{"xxa", "xxx"}, - }, - { - ` -The quick brown fox -jumps over the lazy -dog. -But not quickly. -`[1:], - ` -xxaThe quick brown fox -xxbjumps over the lazy -xxcdog. -xxxBut not quickly. -`[1:], - []string{"xxa", "xxb", "xxc", "xxx"}, - }, - { - ` -The quick brown fox -jumps over the lazy -dog. - -But not quickly.`[1:], - ` -xxaThe quick brown fox -xxxjumps over the lazy -xxxdog. -xxx -xxxBut not quickly.`[1:], - []string{"xxa", "xxx"}, - }, -} - -func TestIndentWriter(t *testing.T) { - for _, test := range ts { - b := new(bytes.Buffer) - pre := make([][]byte, len(test.pre)) - for i := range test.pre { - pre[i] = []byte(test.pre[i]) - } - w := NewIndentWriter(b, pre...) - if _, err := w.Write([]byte(test.inp)); err != nil { - t.Error(err) - } - if got := b.String(); got != test.exp { - t.Errorf("mismatch %q != %q", got, test.exp) - t.Log(got) - t.Log(test.exp) - } - } -} diff --git a/vendor/github.com/kr/text/mc/Readme b/vendor/github.com/kr/text/mc/Readme deleted file mode 100644 index 519ddc00..00000000 --- a/vendor/github.com/kr/text/mc/Readme +++ /dev/null @@ -1,9 +0,0 @@ -Command mc prints in multiple columns. - - Usage: mc [-] [-N] [file...] - -Mc splits the input into as many columns as will fit in N -print positions. If the output is a tty, the default N is -the number of characters in a terminal line; otherwise the -default N is 80. Under option - each input line ending in -a colon ':' is printed separately. diff --git a/vendor/github.com/kr/text/mc/mc.go b/vendor/github.com/kr/text/mc/mc.go deleted file mode 100644 index 00169a30..00000000 --- a/vendor/github.com/kr/text/mc/mc.go +++ /dev/null @@ -1,62 +0,0 @@ -// Command mc prints in multiple columns. -// -// Usage: mc [-] [-N] [file...] -// -// Mc splits the input into as many columns as will fit in N -// print positions. If the output is a tty, the default N is -// the number of characters in a terminal line; otherwise the -// default N is 80. Under option - each input line ending in -// a colon ':' is printed separately. -package main - -import ( - "github.com/kr/pty" - "github.com/kr/text/colwriter" - "io" - "log" - "os" - "strconv" -) - -func main() { - var width int - var flag uint - args := os.Args[1:] - for len(args) > 0 && len(args[0]) > 0 && args[0][0] == '-' { - if len(args[0]) > 1 { - width, _ = strconv.Atoi(args[0][1:]) - } else { - flag |= colwriter.BreakOnColon - } - args = args[1:] - } - if width < 1 { - _, width, _ = pty.Getsize(os.Stdout) - } - if width < 1 { - width = 80 - } - - w := colwriter.NewWriter(os.Stdout, width, flag) - if len(args) > 0 { - for _, s := range args { - if f, err := os.Open(s); err == nil { - copyin(w, f) - f.Close() - } else { - log.Println(err) - } - } - } else { - copyin(w, os.Stdin) - } -} - -func copyin(w *colwriter.Writer, r io.Reader) { - if _, err := io.Copy(w, r); err != nil { - log.Println(err) - } - if err := w.Flush(); err != nil { - log.Println(err) - } -} diff --git a/vendor/github.com/kr/text/wrap.go b/vendor/github.com/kr/text/wrap.go deleted file mode 100644 index b09bb037..00000000 --- a/vendor/github.com/kr/text/wrap.go +++ /dev/null @@ -1,86 +0,0 @@ -package text - -import ( - "bytes" - "math" -) - -var ( - nl = []byte{'\n'} - sp = []byte{' '} -) - -const defaultPenalty = 1e5 - -// Wrap wraps s into a paragraph of lines of length lim, with minimal -// raggedness. -func Wrap(s string, lim int) string { - return string(WrapBytes([]byte(s), lim)) -} - -// WrapBytes wraps b into a paragraph of lines of length lim, with minimal -// raggedness. -func WrapBytes(b []byte, lim int) []byte { - words := bytes.Split(bytes.Replace(bytes.TrimSpace(b), nl, sp, -1), sp) - var lines [][]byte - for _, line := range WrapWords(words, 1, lim, defaultPenalty) { - lines = append(lines, bytes.Join(line, sp)) - } - return bytes.Join(lines, nl) -} - -// WrapWords is the low-level line-breaking algorithm, useful if you need more -// control over the details of the text wrapping process. For most uses, either -// Wrap or WrapBytes will be sufficient and more convenient. -// -// WrapWords splits a list of words into lines with minimal "raggedness", -// treating each byte as one unit, accounting for spc units between adjacent -// words on each line, and attempting to limit lines to lim units. Raggedness -// is the total error over all lines, where error is the square of the -// difference of the length of the line and lim. Too-long lines (which only -// happen when a single word is longer than lim units) have pen penalty units -// added to the error. -func WrapWords(words [][]byte, spc, lim, pen int) [][][]byte { - n := len(words) - - length := make([][]int, n) - for i := 0; i < n; i++ { - length[i] = make([]int, n) - length[i][i] = len(words[i]) - for j := i + 1; j < n; j++ { - length[i][j] = length[i][j-1] + spc + len(words[j]) - } - } - - nbrk := make([]int, n) - cost := make([]int, n) - for i := range cost { - cost[i] = math.MaxInt32 - } - for i := n - 1; i >= 0; i-- { - if length[i][n-1] <= lim || i == n-1 { - cost[i] = 0 - nbrk[i] = n - } else { - for j := i + 1; j < n; j++ { - d := lim - length[i][j-1] - c := d*d + cost[j] - if length[i][j-1] > lim { - c += pen // too-long lines get a worse penalty - } - if c < cost[i] { - cost[i] = c - nbrk[i] = j - } - } - } - } - - var lines [][][]byte - i := 0 - for i < n { - lines = append(lines, words[i:nbrk[i]]) - i = nbrk[i] - } - return lines -} diff --git a/vendor/github.com/kr/text/wrap_test.go b/vendor/github.com/kr/text/wrap_test.go deleted file mode 100644 index 634b6e8e..00000000 --- a/vendor/github.com/kr/text/wrap_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package text - -import ( - "bytes" - "testing" -) - -var text = "The quick brown fox jumps over the lazy dog." - -func TestWrap(t *testing.T) { - exp := [][]string{ - {"The", "quick", "brown", "fox"}, - {"jumps", "over", "the", "lazy", "dog."}, - } - words := bytes.Split([]byte(text), sp) - got := WrapWords(words, 1, 24, defaultPenalty) - if len(exp) != len(got) { - t.Fail() - } - for i := range exp { - if len(exp[i]) != len(got[i]) { - t.Fail() - } - for j := range exp[i] { - if exp[i][j] != string(got[i][j]) { - t.Fatal(i, exp[i][j], got[i][j]) - } - } - } -} - -func TestWrapNarrow(t *testing.T) { - exp := "The\nquick\nbrown\nfox\njumps\nover\nthe\nlazy\ndog." - if Wrap(text, 5) != exp { - t.Fail() - } -} - -func TestWrapOneLine(t *testing.T) { - exp := "The quick brown fox jumps over the lazy dog." - if Wrap(text, 500) != exp { - t.Fail() - } -} - -func TestWrapBug1(t *testing.T) { - cases := []struct { - limit int - text string - want string - }{ - {4, "aaaaa", "aaaaa"}, - {4, "a aaaaa", "a\naaaaa"}, - } - - for _, test := range cases { - got := Wrap(test.text, test.limit) - if got != test.want { - t.Errorf("Wrap(%q, %d) = %q want %q", test.text, test.limit, got, test.want) - } - } -} diff --git a/vendor/github.com/magiconair/properties/CHANGELOG.md b/vendor/github.com/magiconair/properties/CHANGELOG.md index 160cd833..1d86d0c1 100644 --- a/vendor/github.com/magiconair/properties/CHANGELOG.md +++ b/vendor/github.com/magiconair/properties/CHANGELOG.md @@ -1,5 +1,26 @@ ## Changelog +### Unreleased + + * [Issue #17](https://github.com/magiconair/properties/issues/17): Add [SetValue()](http://godoc.org/github.com/magiconair/properties#Properties.SetValue) method to set values generically + * [Issue #22](https://github.com/magiconair/properties/issues/22): Add [LoadMap()](http://godoc.org/github.com/magiconair/properties#LoadMap) function to load properties from a string map + +### [1.7.2](https://github.com/magiconair/properties/tags/v1.7.2) - 20 Mar 2017 + + * [Issue #15](https://github.com/magiconair/properties/issues/15): Drop gocheck dependency + * [PR #21](https://github.com/magiconair/properties/pull/21): Add [Map()](http://godoc.org/github.com/magiconair/properties#Properties.Map) and [FilterFunc()](http://godoc.org/github.com/magiconair/properties#Properties.FilterFunc) + +### [1.7.1](https://github.com/magiconair/properties/tags/v1.7.1) - 13 Jan 2017 + + * [PR #16](https://github.com/magiconair/properties/pull/16): Keep gofmt happy + * [PR #18](https://github.com/magiconair/properties/pull/18): Fix Delete() function + +### [1.7.0](https://github.com/magiconair/properties/tags/v1.7.0) - 20 Mar 2016 + + * [Issue #10](https://github.com/magiconair/properties/issues/10): Add [LoadURL,LoadURLs,MustLoadURL,MustLoadURLs](http://godoc.org/github.com/magiconair/properties#LoadURL) method to load properties from a URL. + * [Issue #11](https://github.com/magiconair/properties/issues/11): Add [LoadString,MustLoadString](http://godoc.org/github.com/magiconair/properties#LoadString) method to load properties from an UTF8 string. + * [PR #8](https://github.com/magiconair/properties/pull/8): Add [MustFlag](http://godoc.org/github.com/magiconair/properties#Properties.MustFlag) method to provide overrides via command line flags. (@pascaldekloe) + ### [1.6.0](https://github.com/magiconair/properties/tags/v1.6.0) - 11 Dec 2015 * Add [Decode](http://godoc.org/github.com/magiconair/properties#Properties.Decode) method to populate struct from properties via tags. @@ -10,7 +31,7 @@ ### [1.5.5](https://github.com/magiconair/properties/tags/v1.5.5) - 31 Jul 2015 - * [Pull Request #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs) + * [PR #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs) ### [1.5.4](https://github.com/magiconair/properties/tags/v1.5.4) - 23 Jun 2015 diff --git a/vendor/github.com/magiconair/properties/README.md b/vendor/github.com/magiconair/properties/README.md index b3b2e5c8..71b6a535 100644 --- a/vendor/github.com/magiconair/properties/README.md +++ b/vendor/github.com/magiconair/properties/README.md @@ -1,16 +1,15 @@ Overview [![Build Status](https://travis-ci.org/magiconair/properties.svg?branch=master)](https://travis-ci.org/magiconair/properties) ======== -#### Current version: 1.6.0 +#### Current version: 1.7.2 properties is a Go library for reading and writing properties files. -It supports reading from multiple files and Spring style recursive property -expansion of expressions like `${key}` to their corresponding value. -Value expressions can refer to other keys like in `${key}` or to -environment variables like in `${USER}`. -Filenames can also contain environment variables like in -`/home/${USER}/myapp.properties`. +It supports reading from multiple files or URLs and Spring style recursive +property expansion of expressions like `${key}` to their corresponding value. +Value expressions can refer to other keys like in `${key}` or to environment +variables like in `${USER}`. Filenames can also contain environment variables +like in `/home/${USER}/myapp.properties`. Properties can be decoded into structs, maps, arrays and values through struct tags. @@ -26,6 +25,8 @@ changed from `panic` to `log.Fatal` but this is configurable and custom error handling functions can be provided. See the package documentation for details. +Read the full documentation on [GoDoc](https://godoc.org/github.com/magiconair/properties) [![GoDoc](https://godoc.org/github.com/magiconair/properties?status.png)](https://godoc.org/github.com/magiconair/properties) + Getting Started --------------- @@ -36,13 +37,38 @@ import ( ) func main() { + // init from a file p := properties.MustLoadFile("${HOME}/config.properties", properties.UTF8) - // via getters + // or multiple files + p = properties.MustLoadFiles([]string{ + "${HOME}/config.properties", + "${HOME}/config-${USER}.properties", + }, properties.UTF8, true) + + // or from a map + p = properties.LoadMap(map[string]string{"key": "value", "abc": "def"}) + + // or from a string + p = properties.MustLoadString("key=value\nabc=def") + + // or from a URL + p = properties.MustLoadURL("http://host/path") + + // or from multiple URLs + p = properties.MustLoadURL([]string{ + "http://host/config", + "http://host/config-${USER}", + }, true) + + // or from flags + p.MustFlag(flag.CommandLine) + + // get values through getters host := p.MustGetString("host") port := p.GetInt("port", 8080) - // or via decode + // or through Decode type Config struct { Host string `properties:"host"` Port int `properties:"port,default=9000"` @@ -53,15 +79,10 @@ func main() { if err := p.Decode(&cfg); err != nil { log.Fatal(err) } - - // or via flags - p.MustFlag(flag.CommandLine) } ``` -Read the full documentation on [GoDoc](https://godoc.org/github.com/magiconair/properties) [![GoDoc](https://godoc.org/github.com/magiconair/properties?status.png)](https://godoc.org/github.com/magiconair/properties) - Installation and Upgrade ------------------------ diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/LICENSE b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/LICENSE deleted file mode 100644 index 545cf2d3..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Gocheck - A rich testing framework for Go - -Copyright (c) 2010-2013 Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/README.md b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/README.md deleted file mode 100644 index 0ca9e572..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/README.md +++ /dev/null @@ -1,20 +0,0 @@ -Instructions -============ - -Install the package with: - - go get gopkg.in/check.v1 - -Import it with: - - import "gopkg.in/check.v1" - -and use _check_ as the package name inside the code. - -For more details, visit the project page: - -* http://labix.org/gocheck - -and the API documentation: - -* https://gopkg.in/check.v1 diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/TODO b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/TODO deleted file mode 100644 index 33498270..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/TODO +++ /dev/null @@ -1,2 +0,0 @@ -- Assert(slice, Contains, item) -- Parallel test support diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark.go deleted file mode 100644 index 48cb8c81..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package check - -import ( - "fmt" - "runtime" - "time" -) - -var memStats runtime.MemStats - -// testingB is a type passed to Benchmark functions to manage benchmark -// timing and to specify the number of iterations to run. -type timer struct { - start time.Time // Time test or benchmark started - duration time.Duration - N int - bytes int64 - timerOn bool - benchTime time.Duration - // The initial states of memStats.Mallocs and memStats.TotalAlloc. - startAllocs uint64 - startBytes uint64 - // The net total of this test after being run. - netAllocs uint64 - netBytes uint64 -} - -// StartTimer starts timing a test. This function is called automatically -// before a benchmark starts, but it can also used to resume timing after -// a call to StopTimer. -func (c *C) StartTimer() { - if !c.timerOn { - c.start = time.Now() - c.timerOn = true - - runtime.ReadMemStats(&memStats) - c.startAllocs = memStats.Mallocs - c.startBytes = memStats.TotalAlloc - } -} - -// StopTimer stops timing a test. This can be used to pause the timer -// while performing complex initialization that you don't -// want to measure. -func (c *C) StopTimer() { - if c.timerOn { - c.duration += time.Now().Sub(c.start) - c.timerOn = false - runtime.ReadMemStats(&memStats) - c.netAllocs += memStats.Mallocs - c.startAllocs - c.netBytes += memStats.TotalAlloc - c.startBytes - } -} - -// ResetTimer sets the elapsed benchmark time to zero. -// It does not affect whether the timer is running. -func (c *C) ResetTimer() { - if c.timerOn { - c.start = time.Now() - runtime.ReadMemStats(&memStats) - c.startAllocs = memStats.Mallocs - c.startBytes = memStats.TotalAlloc - } - c.duration = 0 - c.netAllocs = 0 - c.netBytes = 0 -} - -// SetBytes informs the number of bytes that the benchmark processes -// on each iteration. If this is called in a benchmark it will also -// report MB/s. -func (c *C) SetBytes(n int64) { - c.bytes = n -} - -func (c *C) nsPerOp() int64 { - if c.N <= 0 { - return 0 - } - return c.duration.Nanoseconds() / int64(c.N) -} - -func (c *C) mbPerSec() float64 { - if c.bytes <= 0 || c.duration <= 0 || c.N <= 0 { - return 0 - } - return (float64(c.bytes) * float64(c.N) / 1e6) / c.duration.Seconds() -} - -func (c *C) timerString() string { - if c.N <= 0 { - return fmt.Sprintf("%3.3fs", float64(c.duration.Nanoseconds())/1e9) - } - mbs := c.mbPerSec() - mb := "" - if mbs != 0 { - mb = fmt.Sprintf("\t%7.2f MB/s", mbs) - } - nsop := c.nsPerOp() - ns := fmt.Sprintf("%10d ns/op", nsop) - if c.N > 0 && nsop < 100 { - // The format specifiers here make sure that - // the ones digits line up for all three possible formats. - if nsop < 10 { - ns = fmt.Sprintf("%13.2f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) - } else { - ns = fmt.Sprintf("%12.1f ns/op", float64(c.duration.Nanoseconds())/float64(c.N)) - } - } - memStats := "" - if c.benchMem { - allocedBytes := fmt.Sprintf("%8d B/op", int64(c.netBytes)/int64(c.N)) - allocs := fmt.Sprintf("%8d allocs/op", int64(c.netAllocs)/int64(c.N)) - memStats = fmt.Sprintf("\t%s\t%s", allocedBytes, allocs) - } - return fmt.Sprintf("%8d\t%s%s%s", c.N, ns, mb, memStats) -} - -func min(x, y int) int { - if x > y { - return y - } - return x -} - -func max(x, y int) int { - if x < y { - return y - } - return x -} - -// roundDown10 rounds a number down to the nearest power of 10. -func roundDown10(n int) int { - var tens = 0 - // tens = floor(log_10(n)) - for n > 10 { - n = n / 10 - tens++ - } - // result = 10^tens - result := 1 - for i := 0; i < tens; i++ { - result *= 10 - } - return result -} - -// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. -func roundUp(n int) int { - base := roundDown10(n) - if n < (2 * base) { - return 2 * base - } - if n < (5 * base) { - return 5 * base - } - return 10 * base -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark_test.go deleted file mode 100644 index a7cfd53f..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/benchmark_test.go +++ /dev/null @@ -1,91 +0,0 @@ -// These tests verify the test running logic. - -package check_test - -import ( - . "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" - "time" -) - -var benchmarkS = Suite(&BenchmarkS{}) - -type BenchmarkS struct{} - -func (s *BenchmarkS) TestCountSuite(c *C) { - suitesRun += 1 -} - -func (s *BenchmarkS) TestBasicTestTiming(c *C) { - helper := FixtureHelper{sleepOn: "Test1", sleep: 1000000 * time.Nanosecond} - output := String{} - runConf := RunConf{Output: &output, Verbose: true} - Run(&helper, &runConf) - - expected := "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Test1\t0\\.001s\n" + - "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Test2\t0\\.000s\n" - c.Assert(output.value, Matches, expected) -} - -func (s *BenchmarkS) TestStreamTestTiming(c *C) { - helper := FixtureHelper{sleepOn: "SetUpSuite", sleep: 1000000 * time.Nanosecond} - output := String{} - runConf := RunConf{Output: &output, Stream: true} - Run(&helper, &runConf) - - expected := "(?s).*\nPASS: check_test\\.go:[0-9]+: FixtureHelper\\.SetUpSuite\t *0\\.001s\n.*" - c.Assert(output.value, Matches, expected) -} - -func (s *BenchmarkS) TestBenchmark(c *C) { - helper := FixtureHelper{sleep: 100000} - output := String{} - runConf := RunConf{ - Output: &output, - Benchmark: true, - BenchmarkTime: 10000000, - Filter: "Benchmark1", - } - Run(&helper, &runConf) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Benchmark1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "SetUpTest") - c.Check(helper.calls[5], Equals, "Benchmark1") - c.Check(helper.calls[6], Equals, "TearDownTest") - // ... and more. - - expected := "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Benchmark1\t *100\t *[12][0-9]{5} ns/op\n" - c.Assert(output.value, Matches, expected) -} - -func (s *BenchmarkS) TestBenchmarkBytes(c *C) { - helper := FixtureHelper{sleep: 100000} - output := String{} - runConf := RunConf{ - Output: &output, - Benchmark: true, - BenchmarkTime: 10000000, - Filter: "Benchmark2", - } - Run(&helper, &runConf) - - expected := "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Benchmark2\t *100\t *[12][0-9]{5} ns/op\t *[4-9]\\.[0-9]{2} MB/s\n" - c.Assert(output.value, Matches, expected) -} - -func (s *BenchmarkS) TestBenchmarkMem(c *C) { - helper := FixtureHelper{sleep: 100000} - output := String{} - runConf := RunConf{ - Output: &output, - Benchmark: true, - BenchmarkMem: true, - BenchmarkTime: 10000000, - Filter: "Benchmark3", - } - Run(&helper, &runConf) - - expected := "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Benchmark3\t *100\t *[12][0-9]{5} ns/op\t *[0-9]+ B/op\t *[1-9] allocs/op\n" - c.Assert(output.value, Matches, expected) -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/bootstrap_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/bootstrap_test.go deleted file mode 100644 index e5cee20e..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/bootstrap_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// These initial tests are for bootstrapping. They verify that we can -// basically use the testing infrastructure itself to check if the test -// system is working. -// -// These tests use will break down the test runner badly in case of -// errors because if they simply fail, we can't be sure the developer -// will ever see anything (because failing means the failing system -// somehow isn't working! :-) -// -// Do not assume *any* internal functionality works as expected besides -// what's actually tested here. - -package check_test - -import ( - "fmt" - "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" - "strings" -) - -type BootstrapS struct{} - -var boostrapS = check.Suite(&BootstrapS{}) - -func (s *BootstrapS) TestCountSuite(c *check.C) { - suitesRun += 1 -} - -func (s *BootstrapS) TestFailedAndFail(c *check.C) { - if c.Failed() { - critical("c.Failed() must be false first!") - } - c.Fail() - if !c.Failed() { - critical("c.Fail() didn't put the test in a failed state!") - } - c.Succeed() -} - -func (s *BootstrapS) TestFailedAndSucceed(c *check.C) { - c.Fail() - c.Succeed() - if c.Failed() { - critical("c.Succeed() didn't put the test back in a non-failed state") - } -} - -func (s *BootstrapS) TestLogAndGetTestLog(c *check.C) { - c.Log("Hello there!") - log := c.GetTestLog() - if log != "Hello there!\n" { - critical(fmt.Sprintf("Log() or GetTestLog() is not working! Got: %#v", log)) - } -} - -func (s *BootstrapS) TestLogfAndGetTestLog(c *check.C) { - c.Logf("Hello %v", "there!") - log := c.GetTestLog() - if log != "Hello there!\n" { - critical(fmt.Sprintf("Logf() or GetTestLog() is not working! Got: %#v", log)) - } -} - -func (s *BootstrapS) TestRunShowsErrors(c *check.C) { - output := String{} - check.Run(&FailHelper{}, &check.RunConf{Output: &output}) - if strings.Index(output.value, "Expected failure!") == -1 { - critical(fmt.Sprintf("RunWithWriter() output did not contain the "+ - "expected failure! Got: %#v", - output.value)) - } -} - -func (s *BootstrapS) TestRunDoesntShowSuccesses(c *check.C) { - output := String{} - check.Run(&SuccessHelper{}, &check.RunConf{Output: &output}) - if strings.Index(output.value, "Expected success!") != -1 { - critical(fmt.Sprintf("RunWithWriter() output contained a successful "+ - "test! Got: %#v", - output.value)) - } -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check.go deleted file mode 100644 index ca8c0f92..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check.go +++ /dev/null @@ -1,945 +0,0 @@ -// Package check is a rich testing extension for Go's testing package. -// -// For details about the project, see: -// -// http://labix.org/gocheck -// -package check - -import ( - "bytes" - "errors" - "fmt" - "io" - "math/rand" - "os" - "path" - "path/filepath" - "reflect" - "regexp" - "runtime" - "strconv" - "strings" - "sync" - "time" -) - -// ----------------------------------------------------------------------- -// Internal type which deals with suite method calling. - -const ( - fixtureKd = iota - testKd -) - -type funcKind int - -const ( - succeededSt = iota - failedSt - skippedSt - panickedSt - fixturePanickedSt - missedSt -) - -type funcStatus int - -// A method value can't reach its own Method structure. -type methodType struct { - reflect.Value - Info reflect.Method -} - -func newMethod(receiver reflect.Value, i int) *methodType { - return &methodType{receiver.Method(i), receiver.Type().Method(i)} -} - -func (method *methodType) PC() uintptr { - return method.Info.Func.Pointer() -} - -func (method *methodType) suiteName() string { - t := method.Info.Type.In(0) - if t.Kind() == reflect.Ptr { - t = t.Elem() - } - return t.Name() -} - -func (method *methodType) String() string { - return method.suiteName() + "." + method.Info.Name -} - -func (method *methodType) matches(re *regexp.Regexp) bool { - return (re.MatchString(method.Info.Name) || - re.MatchString(method.suiteName()) || - re.MatchString(method.String())) -} - -type C struct { - method *methodType - kind funcKind - testName string - status funcStatus - logb *logger - logw io.Writer - done chan *C - reason string - mustFail bool - tempDir *tempDir - benchMem bool - startTime time.Time - timer -} - -func (c *C) stopNow() { - runtime.Goexit() -} - -// logger is a concurrency safe byte.Buffer -type logger struct { - sync.Mutex - writer bytes.Buffer -} - -func (l *logger) Write(buf []byte) (int, error) { - l.Lock() - defer l.Unlock() - return l.writer.Write(buf) -} - -func (l *logger) WriteTo(w io.Writer) (int64, error) { - l.Lock() - defer l.Unlock() - return l.writer.WriteTo(w) -} - -func (l *logger) String() string { - l.Lock() - defer l.Unlock() - return l.writer.String() -} - -// ----------------------------------------------------------------------- -// Handling of temporary files and directories. - -type tempDir struct { - sync.Mutex - path string - counter int -} - -func (td *tempDir) newPath() string { - td.Lock() - defer td.Unlock() - if td.path == "" { - var err error - for i := 0; i != 100; i++ { - path := fmt.Sprintf("%s%ccheck-%d", os.TempDir(), os.PathSeparator, rand.Int()) - if err = os.Mkdir(path, 0700); err == nil { - td.path = path - break - } - } - if td.path == "" { - panic("Couldn't create temporary directory: " + err.Error()) - } - } - result := filepath.Join(td.path, strconv.Itoa(td.counter)) - td.counter += 1 - return result -} - -func (td *tempDir) removeAll() { - td.Lock() - defer td.Unlock() - if td.path != "" { - err := os.RemoveAll(td.path) - if err != nil { - fmt.Fprintf(os.Stderr, "WARNING: Error cleaning up temporaries: "+err.Error()) - } - } -} - -// Create a new temporary directory which is automatically removed after -// the suite finishes running. -func (c *C) MkDir() string { - path := c.tempDir.newPath() - if err := os.Mkdir(path, 0700); err != nil { - panic(fmt.Sprintf("Couldn't create temporary directory %s: %s", path, err.Error())) - } - return path -} - -// ----------------------------------------------------------------------- -// Low-level logging functions. - -func (c *C) log(args ...interface{}) { - c.writeLog([]byte(fmt.Sprint(args...) + "\n")) -} - -func (c *C) logf(format string, args ...interface{}) { - c.writeLog([]byte(fmt.Sprintf(format+"\n", args...))) -} - -func (c *C) logNewLine() { - c.writeLog([]byte{'\n'}) -} - -func (c *C) writeLog(buf []byte) { - c.logb.Write(buf) - if c.logw != nil { - c.logw.Write(buf) - } -} - -func hasStringOrError(x interface{}) (ok bool) { - _, ok = x.(fmt.Stringer) - if ok { - return - } - _, ok = x.(error) - return -} - -func (c *C) logValue(label string, value interface{}) { - if label == "" { - if hasStringOrError(value) { - c.logf("... %#v (%q)", value, value) - } else { - c.logf("... %#v", value) - } - } else if value == nil { - c.logf("... %s = nil", label) - } else { - if hasStringOrError(value) { - fv := fmt.Sprintf("%#v", value) - qv := fmt.Sprintf("%q", value) - if fv != qv { - c.logf("... %s %s = %s (%s)", label, reflect.TypeOf(value), fv, qv) - return - } - } - if s, ok := value.(string); ok && isMultiLine(s) { - c.logf(`... %s %s = "" +`, label, reflect.TypeOf(value)) - c.logMultiLine(s) - } else { - c.logf("... %s %s = %#v", label, reflect.TypeOf(value), value) - } - } -} - -func (c *C) logMultiLine(s string) { - b := make([]byte, 0, len(s)*2) - i := 0 - n := len(s) - for i < n { - j := i + 1 - for j < n && s[j-1] != '\n' { - j++ - } - b = append(b, "... "...) - b = strconv.AppendQuote(b, s[i:j]) - if j < n { - b = append(b, " +"...) - } - b = append(b, '\n') - i = j - } - c.writeLog(b) -} - -func isMultiLine(s string) bool { - for i := 0; i+1 < len(s); i++ { - if s[i] == '\n' { - return true - } - } - return false -} - -func (c *C) logString(issue string) { - c.log("... ", issue) -} - -func (c *C) logCaller(skip int) { - // This is a bit heavier than it ought to be. - skip += 1 // Our own frame. - pc, callerFile, callerLine, ok := runtime.Caller(skip) - if !ok { - return - } - var testFile string - var testLine int - testFunc := runtime.FuncForPC(c.method.PC()) - if runtime.FuncForPC(pc) != testFunc { - for { - skip += 1 - if pc, file, line, ok := runtime.Caller(skip); ok { - // Note that the test line may be different on - // distinct calls for the same test. Showing - // the "internal" line is helpful when debugging. - if runtime.FuncForPC(pc) == testFunc { - testFile, testLine = file, line - break - } - } else { - break - } - } - } - if testFile != "" && (testFile != callerFile || testLine != callerLine) { - c.logCode(testFile, testLine) - } - c.logCode(callerFile, callerLine) -} - -func (c *C) logCode(path string, line int) { - c.logf("%s:%d:", nicePath(path), line) - code, err := printLine(path, line) - if code == "" { - code = "..." // XXX Open the file and take the raw line. - if err != nil { - code += err.Error() - } - } - c.log(indent(code, " ")) -} - -var valueGo = filepath.Join("reflect", "value.go") -var asmGo = filepath.Join("runtime", "asm_") - -func (c *C) logPanic(skip int, value interface{}) { - skip++ // Our own frame. - initialSkip := skip - for ; ; skip++ { - if pc, file, line, ok := runtime.Caller(skip); ok { - if skip == initialSkip { - c.logf("... Panic: %s (PC=0x%X)\n", value, pc) - } - name := niceFuncName(pc) - path := nicePath(file) - if strings.Contains(path, "/gopkg.in/check.v") { - continue - } - if name == "Value.call" && strings.HasSuffix(path, valueGo) { - continue - } - if name == "call16" && strings.Contains(path, asmGo) { - continue - } - c.logf("%s:%d\n in %s", nicePath(file), line, name) - } else { - break - } - } -} - -func (c *C) logSoftPanic(issue string) { - c.log("... Panic: ", issue) -} - -func (c *C) logArgPanic(method *methodType, expectedType string) { - c.logf("... Panic: %s argument should be %s", - niceFuncName(method.PC()), expectedType) -} - -// ----------------------------------------------------------------------- -// Some simple formatting helpers. - -var initWD, initWDErr = os.Getwd() - -func init() { - if initWDErr == nil { - initWD = strings.Replace(initWD, "\\", "/", -1) + "/" - } -} - -func nicePath(path string) string { - if initWDErr == nil { - if strings.HasPrefix(path, initWD) { - return path[len(initWD):] - } - } - return path -} - -func niceFuncPath(pc uintptr) string { - function := runtime.FuncForPC(pc) - if function != nil { - filename, line := function.FileLine(pc) - return fmt.Sprintf("%s:%d", nicePath(filename), line) - } - return "" -} - -func niceFuncName(pc uintptr) string { - function := runtime.FuncForPC(pc) - if function != nil { - name := path.Base(function.Name()) - if i := strings.Index(name, "."); i > 0 { - name = name[i+1:] - } - if strings.HasPrefix(name, "(*") { - if i := strings.Index(name, ")"); i > 0 { - name = name[2:i] + name[i+1:] - } - } - if i := strings.LastIndex(name, ".*"); i != -1 { - name = name[:i] + "." + name[i+2:] - } - if i := strings.LastIndex(name, "·"); i != -1 { - name = name[:i] + "." + name[i+2:] - } - return name - } - return "" -} - -// ----------------------------------------------------------------------- -// Result tracker to aggregate call results. - -type Result struct { - Succeeded int - Failed int - Skipped int - Panicked int - FixturePanicked int - ExpectedFailures int - Missed int // Not even tried to run, related to a panic in the fixture. - RunError error // Houston, we've got a problem. - WorkDir string // If KeepWorkDir is true -} - -type resultTracker struct { - result Result - _lastWasProblem bool - _waiting int - _missed int - _expectChan chan *C - _doneChan chan *C - _stopChan chan bool -} - -func newResultTracker() *resultTracker { - return &resultTracker{_expectChan: make(chan *C), // Synchronous - _doneChan: make(chan *C, 32), // Asynchronous - _stopChan: make(chan bool)} // Synchronous -} - -func (tracker *resultTracker) start() { - go tracker._loopRoutine() -} - -func (tracker *resultTracker) waitAndStop() { - <-tracker._stopChan -} - -func (tracker *resultTracker) expectCall(c *C) { - tracker._expectChan <- c -} - -func (tracker *resultTracker) callDone(c *C) { - tracker._doneChan <- c -} - -func (tracker *resultTracker) _loopRoutine() { - for { - var c *C - if tracker._waiting > 0 { - // Calls still running. Can't stop. - select { - // XXX Reindent this (not now to make diff clear) - case c = <-tracker._expectChan: - tracker._waiting += 1 - case c = <-tracker._doneChan: - tracker._waiting -= 1 - switch c.status { - case succeededSt: - if c.kind == testKd { - if c.mustFail { - tracker.result.ExpectedFailures++ - } else { - tracker.result.Succeeded++ - } - } - case failedSt: - tracker.result.Failed++ - case panickedSt: - if c.kind == fixtureKd { - tracker.result.FixturePanicked++ - } else { - tracker.result.Panicked++ - } - case fixturePanickedSt: - // Track it as missed, since the panic - // was on the fixture, not on the test. - tracker.result.Missed++ - case missedSt: - tracker.result.Missed++ - case skippedSt: - if c.kind == testKd { - tracker.result.Skipped++ - } - } - } - } else { - // No calls. Can stop, but no done calls here. - select { - case tracker._stopChan <- true: - return - case c = <-tracker._expectChan: - tracker._waiting += 1 - case c = <-tracker._doneChan: - panic("Tracker got an unexpected done call.") - } - } - } -} - -// ----------------------------------------------------------------------- -// The underlying suite runner. - -type suiteRunner struct { - suite interface{} - setUpSuite, tearDownSuite *methodType - setUpTest, tearDownTest *methodType - tests []*methodType - tracker *resultTracker - tempDir *tempDir - keepDir bool - output *outputWriter - reportedProblemLast bool - benchTime time.Duration - benchMem bool -} - -type RunConf struct { - Output io.Writer - Stream bool - Verbose bool - Filter string - Benchmark bool - BenchmarkTime time.Duration // Defaults to 1 second - BenchmarkMem bool - KeepWorkDir bool -} - -// Create a new suiteRunner able to run all methods in the given suite. -func newSuiteRunner(suite interface{}, runConf *RunConf) *suiteRunner { - var conf RunConf - if runConf != nil { - conf = *runConf - } - if conf.Output == nil { - conf.Output = os.Stdout - } - if conf.Benchmark { - conf.Verbose = true - } - - suiteType := reflect.TypeOf(suite) - suiteNumMethods := suiteType.NumMethod() - suiteValue := reflect.ValueOf(suite) - - runner := &suiteRunner{ - suite: suite, - output: newOutputWriter(conf.Output, conf.Stream, conf.Verbose), - tracker: newResultTracker(), - benchTime: conf.BenchmarkTime, - benchMem: conf.BenchmarkMem, - tempDir: &tempDir{}, - keepDir: conf.KeepWorkDir, - tests: make([]*methodType, 0, suiteNumMethods), - } - if runner.benchTime == 0 { - runner.benchTime = 1 * time.Second - } - - var filterRegexp *regexp.Regexp - if conf.Filter != "" { - if regexp, err := regexp.Compile(conf.Filter); err != nil { - msg := "Bad filter expression: " + err.Error() - runner.tracker.result.RunError = errors.New(msg) - return runner - } else { - filterRegexp = regexp - } - } - - for i := 0; i != suiteNumMethods; i++ { - method := newMethod(suiteValue, i) - switch method.Info.Name { - case "SetUpSuite": - runner.setUpSuite = method - case "TearDownSuite": - runner.tearDownSuite = method - case "SetUpTest": - runner.setUpTest = method - case "TearDownTest": - runner.tearDownTest = method - default: - prefix := "Test" - if conf.Benchmark { - prefix = "Benchmark" - } - if !strings.HasPrefix(method.Info.Name, prefix) { - continue - } - if filterRegexp == nil || method.matches(filterRegexp) { - runner.tests = append(runner.tests, method) - } - } - } - return runner -} - -// Run all methods in the given suite. -func (runner *suiteRunner) run() *Result { - if runner.tracker.result.RunError == nil && len(runner.tests) > 0 { - runner.tracker.start() - if runner.checkFixtureArgs() { - c := runner.runFixture(runner.setUpSuite, "", nil) - if c == nil || c.status == succeededSt { - for i := 0; i != len(runner.tests); i++ { - c := runner.runTest(runner.tests[i]) - if c.status == fixturePanickedSt { - runner.skipTests(missedSt, runner.tests[i+1:]) - break - } - } - } else if c != nil && c.status == skippedSt { - runner.skipTests(skippedSt, runner.tests) - } else { - runner.skipTests(missedSt, runner.tests) - } - runner.runFixture(runner.tearDownSuite, "", nil) - } else { - runner.skipTests(missedSt, runner.tests) - } - runner.tracker.waitAndStop() - if runner.keepDir { - runner.tracker.result.WorkDir = runner.tempDir.path - } else { - runner.tempDir.removeAll() - } - } - return &runner.tracker.result -} - -// Create a call object with the given suite method, and fork a -// goroutine with the provided dispatcher for running it. -func (runner *suiteRunner) forkCall(method *methodType, kind funcKind, testName string, logb *logger, dispatcher func(c *C)) *C { - var logw io.Writer - if runner.output.Stream { - logw = runner.output - } - if logb == nil { - logb = new(logger) - } - c := &C{ - method: method, - kind: kind, - testName: testName, - logb: logb, - logw: logw, - tempDir: runner.tempDir, - done: make(chan *C, 1), - timer: timer{benchTime: runner.benchTime}, - startTime: time.Now(), - benchMem: runner.benchMem, - } - runner.tracker.expectCall(c) - go (func() { - runner.reportCallStarted(c) - defer runner.callDone(c) - dispatcher(c) - })() - return c -} - -// Same as forkCall(), but wait for call to finish before returning. -func (runner *suiteRunner) runFunc(method *methodType, kind funcKind, testName string, logb *logger, dispatcher func(c *C)) *C { - c := runner.forkCall(method, kind, testName, logb, dispatcher) - <-c.done - return c -} - -// Handle a finished call. If there were any panics, update the call status -// accordingly. Then, mark the call as done and report to the tracker. -func (runner *suiteRunner) callDone(c *C) { - value := recover() - if value != nil { - switch v := value.(type) { - case *fixturePanic: - if v.status == skippedSt { - c.status = skippedSt - } else { - c.logSoftPanic("Fixture has panicked (see related PANIC)") - c.status = fixturePanickedSt - } - default: - c.logPanic(1, value) - c.status = panickedSt - } - } - if c.mustFail { - switch c.status { - case failedSt: - c.status = succeededSt - case succeededSt: - c.status = failedSt - c.logString("Error: Test succeeded, but was expected to fail") - c.logString("Reason: " + c.reason) - } - } - - runner.reportCallDone(c) - c.done <- c -} - -// Runs a fixture call synchronously. The fixture will still be run in a -// goroutine like all suite methods, but this method will not return -// while the fixture goroutine is not done, because the fixture must be -// run in a desired order. -func (runner *suiteRunner) runFixture(method *methodType, testName string, logb *logger) *C { - if method != nil { - c := runner.runFunc(method, fixtureKd, testName, logb, func(c *C) { - c.ResetTimer() - c.StartTimer() - defer c.StopTimer() - c.method.Call([]reflect.Value{reflect.ValueOf(c)}) - }) - return c - } - return nil -} - -// Run the fixture method with runFixture(), but panic with a fixturePanic{} -// in case the fixture method panics. This makes it easier to track the -// fixture panic together with other call panics within forkTest(). -func (runner *suiteRunner) runFixtureWithPanic(method *methodType, testName string, logb *logger, skipped *bool) *C { - if skipped != nil && *skipped { - return nil - } - c := runner.runFixture(method, testName, logb) - if c != nil && c.status != succeededSt { - if skipped != nil { - *skipped = c.status == skippedSt - } - panic(&fixturePanic{c.status, method}) - } - return c -} - -type fixturePanic struct { - status funcStatus - method *methodType -} - -// Run the suite test method, together with the test-specific fixture, -// asynchronously. -func (runner *suiteRunner) forkTest(method *methodType) *C { - testName := method.String() - return runner.forkCall(method, testKd, testName, nil, func(c *C) { - var skipped bool - defer runner.runFixtureWithPanic(runner.tearDownTest, testName, nil, &skipped) - defer c.StopTimer() - benchN := 1 - for { - runner.runFixtureWithPanic(runner.setUpTest, testName, c.logb, &skipped) - mt := c.method.Type() - if mt.NumIn() != 1 || mt.In(0) != reflect.TypeOf(c) { - // Rather than a plain panic, provide a more helpful message when - // the argument type is incorrect. - c.status = panickedSt - c.logArgPanic(c.method, "*check.C") - return - } - if strings.HasPrefix(c.method.Info.Name, "Test") { - c.ResetTimer() - c.StartTimer() - c.method.Call([]reflect.Value{reflect.ValueOf(c)}) - return - } - if !strings.HasPrefix(c.method.Info.Name, "Benchmark") { - panic("unexpected method prefix: " + c.method.Info.Name) - } - - runtime.GC() - c.N = benchN - c.ResetTimer() - c.StartTimer() - c.method.Call([]reflect.Value{reflect.ValueOf(c)}) - c.StopTimer() - if c.status != succeededSt || c.duration >= c.benchTime || benchN >= 1e9 { - return - } - perOpN := int(1e9) - if c.nsPerOp() != 0 { - perOpN = int(c.benchTime.Nanoseconds() / c.nsPerOp()) - } - - // Logic taken from the stock testing package: - // - Run more iterations than we think we'll need for a second (1.5x). - // - Don't grow too fast in case we had timing errors previously. - // - Be sure to run at least one more than last time. - benchN = max(min(perOpN+perOpN/2, 100*benchN), benchN+1) - benchN = roundUp(benchN) - - skipped = true // Don't run the deferred one if this panics. - runner.runFixtureWithPanic(runner.tearDownTest, testName, nil, nil) - skipped = false - } - }) -} - -// Same as forkTest(), but wait for the test to finish before returning. -func (runner *suiteRunner) runTest(method *methodType) *C { - c := runner.forkTest(method) - <-c.done - return c -} - -// Helper to mark tests as skipped or missed. A bit heavy for what -// it does, but it enables homogeneous handling of tracking, including -// nice verbose output. -func (runner *suiteRunner) skipTests(status funcStatus, methods []*methodType) { - for _, method := range methods { - runner.runFunc(method, testKd, "", nil, func(c *C) { - c.status = status - }) - } -} - -// Verify if the fixture arguments are *check.C. In case of errors, -// log the error as a panic in the fixture method call, and return false. -func (runner *suiteRunner) checkFixtureArgs() bool { - succeeded := true - argType := reflect.TypeOf(&C{}) - for _, method := range []*methodType{runner.setUpSuite, runner.tearDownSuite, runner.setUpTest, runner.tearDownTest} { - if method != nil { - mt := method.Type() - if mt.NumIn() != 1 || mt.In(0) != argType { - succeeded = false - runner.runFunc(method, fixtureKd, "", nil, func(c *C) { - c.logArgPanic(method, "*check.C") - c.status = panickedSt - }) - } - } - } - return succeeded -} - -func (runner *suiteRunner) reportCallStarted(c *C) { - runner.output.WriteCallStarted("START", c) -} - -func (runner *suiteRunner) reportCallDone(c *C) { - runner.tracker.callDone(c) - switch c.status { - case succeededSt: - if c.mustFail { - runner.output.WriteCallSuccess("FAIL EXPECTED", c) - } else { - runner.output.WriteCallSuccess("PASS", c) - } - case skippedSt: - runner.output.WriteCallSuccess("SKIP", c) - case failedSt: - runner.output.WriteCallProblem("FAIL", c) - case panickedSt: - runner.output.WriteCallProblem("PANIC", c) - case fixturePanickedSt: - // That's a testKd call reporting that its fixture - // has panicked. The fixture call which caused the - // panic itself was tracked above. We'll report to - // aid debugging. - runner.output.WriteCallProblem("PANIC", c) - case missedSt: - runner.output.WriteCallSuccess("MISS", c) - } -} - -// ----------------------------------------------------------------------- -// Output writer manages atomic output writing according to settings. - -type outputWriter struct { - m sync.Mutex - writer io.Writer - wroteCallProblemLast bool - Stream bool - Verbose bool -} - -func newOutputWriter(writer io.Writer, stream, verbose bool) *outputWriter { - return &outputWriter{writer: writer, Stream: stream, Verbose: verbose} -} - -func (ow *outputWriter) Write(content []byte) (n int, err error) { - ow.m.Lock() - n, err = ow.writer.Write(content) - ow.m.Unlock() - return -} - -func (ow *outputWriter) WriteCallStarted(label string, c *C) { - if ow.Stream { - header := renderCallHeader(label, c, "", "\n") - ow.m.Lock() - ow.writer.Write([]byte(header)) - ow.m.Unlock() - } -} - -func (ow *outputWriter) WriteCallProblem(label string, c *C) { - var prefix string - if !ow.Stream { - prefix = "\n-----------------------------------" + - "-----------------------------------\n" - } - header := renderCallHeader(label, c, prefix, "\n\n") - ow.m.Lock() - ow.wroteCallProblemLast = true - ow.writer.Write([]byte(header)) - if !ow.Stream { - c.logb.WriteTo(ow.writer) - } - ow.m.Unlock() -} - -func (ow *outputWriter) WriteCallSuccess(label string, c *C) { - if ow.Stream || (ow.Verbose && c.kind == testKd) { - // TODO Use a buffer here. - var suffix string - if c.reason != "" { - suffix = " (" + c.reason + ")" - } - if c.status == succeededSt { - suffix += "\t" + c.timerString() - } - suffix += "\n" - if ow.Stream { - suffix += "\n" - } - header := renderCallHeader(label, c, "", suffix) - ow.m.Lock() - // Resist temptation of using line as prefix above due to race. - if !ow.Stream && ow.wroteCallProblemLast { - header = "\n-----------------------------------" + - "-----------------------------------\n" + - header - } - ow.wroteCallProblemLast = false - ow.writer.Write([]byte(header)) - ow.m.Unlock() - } -} - -func renderCallHeader(label string, c *C, prefix, suffix string) string { - pc := c.method.PC() - return fmt.Sprintf("%s%s: %s: %s%s", prefix, label, niceFuncPath(pc), - niceFuncName(pc), suffix) -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check_test.go deleted file mode 100644 index 2fb8f897..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/check_test.go +++ /dev/null @@ -1,207 +0,0 @@ -// This file contains just a few generic helpers which are used by the -// other test files. - -package check_test - -import ( - "flag" - "fmt" - "os" - "regexp" - "runtime" - "testing" - "time" - - "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" -) - -// We count the number of suites run at least to get a vague hint that the -// test suite is behaving as it should. Otherwise a bug introduced at the -// very core of the system could go unperceived. -const suitesRunExpected = 8 - -var suitesRun int = 0 - -func Test(t *testing.T) { - check.TestingT(t) - if suitesRun != suitesRunExpected && flag.Lookup("check.f").Value.String() == "" { - critical(fmt.Sprintf("Expected %d suites to run rather than %d", - suitesRunExpected, suitesRun)) - } -} - -// ----------------------------------------------------------------------- -// Helper functions. - -// Break down badly. This is used in test cases which can't yet assume -// that the fundamental bits are working. -func critical(error string) { - fmt.Fprintln(os.Stderr, "CRITICAL: "+error) - os.Exit(1) -} - -// Return the file line where it's called. -func getMyLine() int { - if _, _, line, ok := runtime.Caller(1); ok { - return line - } - return -1 -} - -// ----------------------------------------------------------------------- -// Helper type implementing a basic io.Writer for testing output. - -// Type implementing the io.Writer interface for analyzing output. -type String struct { - value string -} - -// The only function required by the io.Writer interface. Will append -// written data to the String.value string. -func (s *String) Write(p []byte) (n int, err error) { - s.value += string(p) - return len(p), nil -} - -// Trivial wrapper to test errors happening on a different file -// than the test itself. -func checkEqualWrapper(c *check.C, obtained, expected interface{}) (result bool, line int) { - return c.Check(obtained, check.Equals, expected), getMyLine() -} - -// ----------------------------------------------------------------------- -// Helper suite for testing basic fail behavior. - -type FailHelper struct { - testLine int -} - -func (s *FailHelper) TestLogAndFail(c *check.C) { - s.testLine = getMyLine() - 1 - c.Log("Expected failure!") - c.Fail() -} - -// ----------------------------------------------------------------------- -// Helper suite for testing basic success behavior. - -type SuccessHelper struct{} - -func (s *SuccessHelper) TestLogAndSucceed(c *check.C) { - c.Log("Expected success!") -} - -// ----------------------------------------------------------------------- -// Helper suite for testing ordering and behavior of fixture. - -type FixtureHelper struct { - calls []string - panicOn string - skip bool - skipOnN int - sleepOn string - sleep time.Duration - bytes int64 -} - -func (s *FixtureHelper) trace(name string, c *check.C) { - s.calls = append(s.calls, name) - if name == s.panicOn { - panic(name) - } - if s.sleep > 0 && s.sleepOn == name { - time.Sleep(s.sleep) - } - if s.skip && s.skipOnN == len(s.calls)-1 { - c.Skip("skipOnN == n") - } -} - -func (s *FixtureHelper) SetUpSuite(c *check.C) { - s.trace("SetUpSuite", c) -} - -func (s *FixtureHelper) TearDownSuite(c *check.C) { - s.trace("TearDownSuite", c) -} - -func (s *FixtureHelper) SetUpTest(c *check.C) { - s.trace("SetUpTest", c) -} - -func (s *FixtureHelper) TearDownTest(c *check.C) { - s.trace("TearDownTest", c) -} - -func (s *FixtureHelper) Test1(c *check.C) { - s.trace("Test1", c) -} - -func (s *FixtureHelper) Test2(c *check.C) { - s.trace("Test2", c) -} - -func (s *FixtureHelper) Benchmark1(c *check.C) { - s.trace("Benchmark1", c) - for i := 0; i < c.N; i++ { - time.Sleep(s.sleep) - } -} - -func (s *FixtureHelper) Benchmark2(c *check.C) { - s.trace("Benchmark2", c) - c.SetBytes(1024) - for i := 0; i < c.N; i++ { - time.Sleep(s.sleep) - } -} - -func (s *FixtureHelper) Benchmark3(c *check.C) { - var x []int64 - s.trace("Benchmark3", c) - for i := 0; i < c.N; i++ { - time.Sleep(s.sleep) - x = make([]int64, 5) - _ = x - } -} - -// ----------------------------------------------------------------------- -// Helper which checks the state of the test and ensures that it matches -// the given expectations. Depends on c.Errorf() working, so shouldn't -// be used to test this one function. - -type expectedState struct { - name string - result interface{} - failed bool - log string -} - -// Verify the state of the test. Note that since this also verifies if -// the test is supposed to be in a failed state, no other checks should -// be done in addition to what is being tested. -func checkState(c *check.C, result interface{}, expected *expectedState) { - failed := c.Failed() - c.Succeed() - log := c.GetTestLog() - matched, matchError := regexp.MatchString("^"+expected.log+"$", log) - if matchError != nil { - c.Errorf("Error in matching expression used in testing %s", - expected.name) - } else if !matched { - c.Errorf("%s logged:\n----------\n%s----------\n\nExpected:\n----------\n%s\n----------", - expected.name, log, expected.log) - } - if result != expected.result { - c.Errorf("%s returned %#v rather than %#v", - expected.name, result, expected.result) - } - if failed != expected.failed { - if failed { - c.Errorf("%s has failed when it shouldn't", expected.name) - } else { - c.Errorf("%s has not failed when it should", expected.name) - } - } -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers.go deleted file mode 100644 index bac33872..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers.go +++ /dev/null @@ -1,458 +0,0 @@ -package check - -import ( - "fmt" - "reflect" - "regexp" -) - -// ----------------------------------------------------------------------- -// CommentInterface and Commentf helper, to attach extra information to checks. - -type comment struct { - format string - args []interface{} -} - -// Commentf returns an infomational value to use with Assert or Check calls. -// If the checker test fails, the provided arguments will be passed to -// fmt.Sprintf, and will be presented next to the logged failure. -// -// For example: -// -// c.Assert(v, Equals, 42, Commentf("Iteration #%d failed.", i)) -// -// Note that if the comment is constant, a better option is to -// simply use a normal comment right above or next to the line, as -// it will also get printed with any errors: -// -// c.Assert(l, Equals, 8192) // Ensure buffer size is correct (bug #123) -// -func Commentf(format string, args ...interface{}) CommentInterface { - return &comment{format, args} -} - -// CommentInterface must be implemented by types that attach extra -// information to failed checks. See the Commentf function for details. -type CommentInterface interface { - CheckCommentString() string -} - -func (c *comment) CheckCommentString() string { - return fmt.Sprintf(c.format, c.args...) -} - -// ----------------------------------------------------------------------- -// The Checker interface. - -// The Checker interface must be provided by checkers used with -// the Assert and Check verification methods. -type Checker interface { - Info() *CheckerInfo - Check(params []interface{}, names []string) (result bool, error string) -} - -// See the Checker interface. -type CheckerInfo struct { - Name string - Params []string -} - -func (info *CheckerInfo) Info() *CheckerInfo { - return info -} - -// ----------------------------------------------------------------------- -// Not checker logic inverter. - -// The Not checker inverts the logic of the provided checker. The -// resulting checker will succeed where the original one failed, and -// vice-versa. -// -// For example: -// -// c.Assert(a, Not(Equals), b) -// -func Not(checker Checker) Checker { - return ¬Checker{checker} -} - -type notChecker struct { - sub Checker -} - -func (checker *notChecker) Info() *CheckerInfo { - info := *checker.sub.Info() - info.Name = "Not(" + info.Name + ")" - return &info -} - -func (checker *notChecker) Check(params []interface{}, names []string) (result bool, error string) { - result, error = checker.sub.Check(params, names) - result = !result - return -} - -// ----------------------------------------------------------------------- -// IsNil checker. - -type isNilChecker struct { - *CheckerInfo -} - -// The IsNil checker tests whether the obtained value is nil. -// -// For example: -// -// c.Assert(err, IsNil) -// -var IsNil Checker = &isNilChecker{ - &CheckerInfo{Name: "IsNil", Params: []string{"value"}}, -} - -func (checker *isNilChecker) Check(params []interface{}, names []string) (result bool, error string) { - return isNil(params[0]), "" -} - -func isNil(obtained interface{}) (result bool) { - if obtained == nil { - result = true - } else { - switch v := reflect.ValueOf(obtained); v.Kind() { - case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice: - return v.IsNil() - } - } - return -} - -// ----------------------------------------------------------------------- -// NotNil checker. Alias for Not(IsNil), since it's so common. - -type notNilChecker struct { - *CheckerInfo -} - -// The NotNil checker verifies that the obtained value is not nil. -// -// For example: -// -// c.Assert(iface, NotNil) -// -// This is an alias for Not(IsNil), made available since it's a -// fairly common check. -// -var NotNil Checker = ¬NilChecker{ - &CheckerInfo{Name: "NotNil", Params: []string{"value"}}, -} - -func (checker *notNilChecker) Check(params []interface{}, names []string) (result bool, error string) { - return !isNil(params[0]), "" -} - -// ----------------------------------------------------------------------- -// Equals checker. - -type equalsChecker struct { - *CheckerInfo -} - -// The Equals checker verifies that the obtained value is equal to -// the expected value, according to usual Go semantics for ==. -// -// For example: -// -// c.Assert(value, Equals, 42) -// -var Equals Checker = &equalsChecker{ - &CheckerInfo{Name: "Equals", Params: []string{"obtained", "expected"}}, -} - -func (checker *equalsChecker) Check(params []interface{}, names []string) (result bool, error string) { - defer func() { - if v := recover(); v != nil { - result = false - error = fmt.Sprint(v) - } - }() - return params[0] == params[1], "" -} - -// ----------------------------------------------------------------------- -// DeepEquals checker. - -type deepEqualsChecker struct { - *CheckerInfo -} - -// The DeepEquals checker verifies that the obtained value is deep-equal to -// the expected value. The check will work correctly even when facing -// slices, interfaces, and values of different types (which always fail -// the test). -// -// For example: -// -// c.Assert(value, DeepEquals, 42) -// c.Assert(array, DeepEquals, []string{"hi", "there"}) -// -var DeepEquals Checker = &deepEqualsChecker{ - &CheckerInfo{Name: "DeepEquals", Params: []string{"obtained", "expected"}}, -} - -func (checker *deepEqualsChecker) Check(params []interface{}, names []string) (result bool, error string) { - return reflect.DeepEqual(params[0], params[1]), "" -} - -// ----------------------------------------------------------------------- -// HasLen checker. - -type hasLenChecker struct { - *CheckerInfo -} - -// The HasLen checker verifies that the obtained value has the -// provided length. In many cases this is superior to using Equals -// in conjuction with the len function because in case the check -// fails the value itself will be printed, instead of its length, -// providing more details for figuring the problem. -// -// For example: -// -// c.Assert(list, HasLen, 5) -// -var HasLen Checker = &hasLenChecker{ - &CheckerInfo{Name: "HasLen", Params: []string{"obtained", "n"}}, -} - -func (checker *hasLenChecker) Check(params []interface{}, names []string) (result bool, error string) { - n, ok := params[1].(int) - if !ok { - return false, "n must be an int" - } - value := reflect.ValueOf(params[0]) - switch value.Kind() { - case reflect.Map, reflect.Array, reflect.Slice, reflect.Chan, reflect.String: - default: - return false, "obtained value type has no length" - } - return value.Len() == n, "" -} - -// ----------------------------------------------------------------------- -// ErrorMatches checker. - -type errorMatchesChecker struct { - *CheckerInfo -} - -// The ErrorMatches checker verifies that the error value -// is non nil and matches the regular expression provided. -// -// For example: -// -// c.Assert(err, ErrorMatches, "perm.*denied") -// -var ErrorMatches Checker = errorMatchesChecker{ - &CheckerInfo{Name: "ErrorMatches", Params: []string{"value", "regex"}}, -} - -func (checker errorMatchesChecker) Check(params []interface{}, names []string) (result bool, errStr string) { - if params[0] == nil { - return false, "Error value is nil" - } - err, ok := params[0].(error) - if !ok { - return false, "Value is not an error" - } - params[0] = err.Error() - names[0] = "error" - return matches(params[0], params[1]) -} - -// ----------------------------------------------------------------------- -// Matches checker. - -type matchesChecker struct { - *CheckerInfo -} - -// The Matches checker verifies that the string provided as the obtained -// value (or the string resulting from obtained.String()) matches the -// regular expression provided. -// -// For example: -// -// c.Assert(err, Matches, "perm.*denied") -// -var Matches Checker = &matchesChecker{ - &CheckerInfo{Name: "Matches", Params: []string{"value", "regex"}}, -} - -func (checker *matchesChecker) Check(params []interface{}, names []string) (result bool, error string) { - return matches(params[0], params[1]) -} - -func matches(value, regex interface{}) (result bool, error string) { - reStr, ok := regex.(string) - if !ok { - return false, "Regex must be a string" - } - valueStr, valueIsStr := value.(string) - if !valueIsStr { - if valueWithStr, valueHasStr := value.(fmt.Stringer); valueHasStr { - valueStr, valueIsStr = valueWithStr.String(), true - } - } - if valueIsStr { - matches, err := regexp.MatchString("^"+reStr+"$", valueStr) - if err != nil { - return false, "Can't compile regex: " + err.Error() - } - return matches, "" - } - return false, "Obtained value is not a string and has no .String()" -} - -// ----------------------------------------------------------------------- -// Panics checker. - -type panicsChecker struct { - *CheckerInfo -} - -// The Panics checker verifies that calling the provided zero-argument -// function will cause a panic which is deep-equal to the provided value. -// -// For example: -// -// c.Assert(func() { f(1, 2) }, Panics, &SomeErrorType{"BOOM"}). -// -// -var Panics Checker = &panicsChecker{ - &CheckerInfo{Name: "Panics", Params: []string{"function", "expected"}}, -} - -func (checker *panicsChecker) Check(params []interface{}, names []string) (result bool, error string) { - f := reflect.ValueOf(params[0]) - if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { - return false, "Function must take zero arguments" - } - defer func() { - // If the function has not panicked, then don't do the check. - if error != "" { - return - } - params[0] = recover() - names[0] = "panic" - result = reflect.DeepEqual(params[0], params[1]) - }() - f.Call(nil) - return false, "Function has not panicked" -} - -type panicMatchesChecker struct { - *CheckerInfo -} - -// The PanicMatches checker verifies that calling the provided zero-argument -// function will cause a panic with an error value matching -// the regular expression provided. -// -// For example: -// -// c.Assert(func() { f(1, 2) }, PanicMatches, `open.*: no such file or directory`). -// -// -var PanicMatches Checker = &panicMatchesChecker{ - &CheckerInfo{Name: "PanicMatches", Params: []string{"function", "expected"}}, -} - -func (checker *panicMatchesChecker) Check(params []interface{}, names []string) (result bool, errmsg string) { - f := reflect.ValueOf(params[0]) - if f.Kind() != reflect.Func || f.Type().NumIn() != 0 { - return false, "Function must take zero arguments" - } - defer func() { - // If the function has not panicked, then don't do the check. - if errmsg != "" { - return - } - obtained := recover() - names[0] = "panic" - if e, ok := obtained.(error); ok { - params[0] = e.Error() - } else if _, ok := obtained.(string); ok { - params[0] = obtained - } else { - errmsg = "Panic value is not a string or an error" - return - } - result, errmsg = matches(params[0], params[1]) - }() - f.Call(nil) - return false, "Function has not panicked" -} - -// ----------------------------------------------------------------------- -// FitsTypeOf checker. - -type fitsTypeChecker struct { - *CheckerInfo -} - -// The FitsTypeOf checker verifies that the obtained value is -// assignable to a variable with the same type as the provided -// sample value. -// -// For example: -// -// c.Assert(value, FitsTypeOf, int64(0)) -// c.Assert(value, FitsTypeOf, os.Error(nil)) -// -var FitsTypeOf Checker = &fitsTypeChecker{ - &CheckerInfo{Name: "FitsTypeOf", Params: []string{"obtained", "sample"}}, -} - -func (checker *fitsTypeChecker) Check(params []interface{}, names []string) (result bool, error string) { - obtained := reflect.ValueOf(params[0]) - sample := reflect.ValueOf(params[1]) - if !obtained.IsValid() { - return false, "" - } - if !sample.IsValid() { - return false, "Invalid sample value" - } - return obtained.Type().AssignableTo(sample.Type()), "" -} - -// ----------------------------------------------------------------------- -// Implements checker. - -type implementsChecker struct { - *CheckerInfo -} - -// The Implements checker verifies that the obtained value -// implements the interface specified via a pointer to an interface -// variable. -// -// For example: -// -// var e os.Error -// c.Assert(err, Implements, &e) -// -var Implements Checker = &implementsChecker{ - &CheckerInfo{Name: "Implements", Params: []string{"obtained", "ifaceptr"}}, -} - -func (checker *implementsChecker) Check(params []interface{}, names []string) (result bool, error string) { - obtained := reflect.ValueOf(params[0]) - ifaceptr := reflect.ValueOf(params[1]) - if !obtained.IsValid() { - return false, "" - } - if !ifaceptr.IsValid() || ifaceptr.Kind() != reflect.Ptr || ifaceptr.Elem().Kind() != reflect.Interface { - return false, "ifaceptr should be a pointer to an interface variable" - } - return obtained.Type().Implements(ifaceptr.Elem().Type()), "" -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers_test.go deleted file mode 100644 index 2da898af..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/checkers_test.go +++ /dev/null @@ -1,272 +0,0 @@ -package check_test - -import ( - "errors" - "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" - "reflect" - "runtime" -) - -type CheckersS struct{} - -var _ = check.Suite(&CheckersS{}) - -func testInfo(c *check.C, checker check.Checker, name string, paramNames []string) { - info := checker.Info() - if info.Name != name { - c.Fatalf("Got name %s, expected %s", info.Name, name) - } - if !reflect.DeepEqual(info.Params, paramNames) { - c.Fatalf("Got param names %#v, expected %#v", info.Params, paramNames) - } -} - -func testCheck(c *check.C, checker check.Checker, result bool, error string, params ...interface{}) ([]interface{}, []string) { - info := checker.Info() - if len(params) != len(info.Params) { - c.Fatalf("unexpected param count in test; expected %d got %d", len(info.Params), len(params)) - } - names := append([]string{}, info.Params...) - result_, error_ := checker.Check(params, names) - if result_ != result || error_ != error { - c.Fatalf("%s.Check(%#v) returned (%#v, %#v) rather than (%#v, %#v)", - info.Name, params, result_, error_, result, error) - } - return params, names -} - -func (s *CheckersS) TestComment(c *check.C) { - bug := check.Commentf("a %d bc", 42) - comment := bug.CheckCommentString() - if comment != "a 42 bc" { - c.Fatalf("Commentf returned %#v", comment) - } -} - -func (s *CheckersS) TestIsNil(c *check.C) { - testInfo(c, check.IsNil, "IsNil", []string{"value"}) - - testCheck(c, check.IsNil, true, "", nil) - testCheck(c, check.IsNil, false, "", "a") - - testCheck(c, check.IsNil, true, "", (chan int)(nil)) - testCheck(c, check.IsNil, false, "", make(chan int)) - testCheck(c, check.IsNil, true, "", (error)(nil)) - testCheck(c, check.IsNil, false, "", errors.New("")) - testCheck(c, check.IsNil, true, "", ([]int)(nil)) - testCheck(c, check.IsNil, false, "", make([]int, 1)) - testCheck(c, check.IsNil, false, "", int(0)) -} - -func (s *CheckersS) TestNotNil(c *check.C) { - testInfo(c, check.NotNil, "NotNil", []string{"value"}) - - testCheck(c, check.NotNil, false, "", nil) - testCheck(c, check.NotNil, true, "", "a") - - testCheck(c, check.NotNil, false, "", (chan int)(nil)) - testCheck(c, check.NotNil, true, "", make(chan int)) - testCheck(c, check.NotNil, false, "", (error)(nil)) - testCheck(c, check.NotNil, true, "", errors.New("")) - testCheck(c, check.NotNil, false, "", ([]int)(nil)) - testCheck(c, check.NotNil, true, "", make([]int, 1)) -} - -func (s *CheckersS) TestNot(c *check.C) { - testInfo(c, check.Not(check.IsNil), "Not(IsNil)", []string{"value"}) - - testCheck(c, check.Not(check.IsNil), false, "", nil) - testCheck(c, check.Not(check.IsNil), true, "", "a") -} - -type simpleStruct struct { - i int -} - -func (s *CheckersS) TestEquals(c *check.C) { - testInfo(c, check.Equals, "Equals", []string{"obtained", "expected"}) - - // The simplest. - testCheck(c, check.Equals, true, "", 42, 42) - testCheck(c, check.Equals, false, "", 42, 43) - - // Different native types. - testCheck(c, check.Equals, false, "", int32(42), int64(42)) - - // With nil. - testCheck(c, check.Equals, false, "", 42, nil) - - // Slices - testCheck(c, check.Equals, false, "runtime error: comparing uncomparable type []uint8", []byte{1, 2}, []byte{1, 2}) - - // Struct values - testCheck(c, check.Equals, true, "", simpleStruct{1}, simpleStruct{1}) - testCheck(c, check.Equals, false, "", simpleStruct{1}, simpleStruct{2}) - - // Struct pointers - testCheck(c, check.Equals, false, "", &simpleStruct{1}, &simpleStruct{1}) - testCheck(c, check.Equals, false, "", &simpleStruct{1}, &simpleStruct{2}) -} - -func (s *CheckersS) TestDeepEquals(c *check.C) { - testInfo(c, check.DeepEquals, "DeepEquals", []string{"obtained", "expected"}) - - // The simplest. - testCheck(c, check.DeepEquals, true, "", 42, 42) - testCheck(c, check.DeepEquals, false, "", 42, 43) - - // Different native types. - testCheck(c, check.DeepEquals, false, "", int32(42), int64(42)) - - // With nil. - testCheck(c, check.DeepEquals, false, "", 42, nil) - - // Slices - testCheck(c, check.DeepEquals, true, "", []byte{1, 2}, []byte{1, 2}) - testCheck(c, check.DeepEquals, false, "", []byte{1, 2}, []byte{1, 3}) - - // Struct values - testCheck(c, check.DeepEquals, true, "", simpleStruct{1}, simpleStruct{1}) - testCheck(c, check.DeepEquals, false, "", simpleStruct{1}, simpleStruct{2}) - - // Struct pointers - testCheck(c, check.DeepEquals, true, "", &simpleStruct{1}, &simpleStruct{1}) - testCheck(c, check.DeepEquals, false, "", &simpleStruct{1}, &simpleStruct{2}) -} - -func (s *CheckersS) TestHasLen(c *check.C) { - testInfo(c, check.HasLen, "HasLen", []string{"obtained", "n"}) - - testCheck(c, check.HasLen, true, "", "abcd", 4) - testCheck(c, check.HasLen, true, "", []int{1, 2}, 2) - testCheck(c, check.HasLen, false, "", []int{1, 2}, 3) - - testCheck(c, check.HasLen, false, "n must be an int", []int{1, 2}, "2") - testCheck(c, check.HasLen, false, "obtained value type has no length", nil, 2) -} - -func (s *CheckersS) TestErrorMatches(c *check.C) { - testInfo(c, check.ErrorMatches, "ErrorMatches", []string{"value", "regex"}) - - testCheck(c, check.ErrorMatches, false, "Error value is nil", nil, "some error") - testCheck(c, check.ErrorMatches, false, "Value is not an error", 1, "some error") - testCheck(c, check.ErrorMatches, true, "", errors.New("some error"), "some error") - testCheck(c, check.ErrorMatches, true, "", errors.New("some error"), "so.*or") - - // Verify params mutation - params, names := testCheck(c, check.ErrorMatches, false, "", errors.New("some error"), "other error") - c.Assert(params[0], check.Equals, "some error") - c.Assert(names[0], check.Equals, "error") -} - -func (s *CheckersS) TestMatches(c *check.C) { - testInfo(c, check.Matches, "Matches", []string{"value", "regex"}) - - // Simple matching - testCheck(c, check.Matches, true, "", "abc", "abc") - testCheck(c, check.Matches, true, "", "abc", "a.c") - - // Must match fully - testCheck(c, check.Matches, false, "", "abc", "ab") - testCheck(c, check.Matches, false, "", "abc", "bc") - - // String()-enabled values accepted - testCheck(c, check.Matches, true, "", reflect.ValueOf("abc"), "a.c") - testCheck(c, check.Matches, false, "", reflect.ValueOf("abc"), "a.d") - - // Some error conditions. - testCheck(c, check.Matches, false, "Obtained value is not a string and has no .String()", 1, "a.c") - testCheck(c, check.Matches, false, "Can't compile regex: error parsing regexp: missing closing ]: `[c$`", "abc", "a[c") -} - -func (s *CheckersS) TestPanics(c *check.C) { - testInfo(c, check.Panics, "Panics", []string{"function", "expected"}) - - // Some errors. - testCheck(c, check.Panics, false, "Function has not panicked", func() bool { return false }, "BOOM") - testCheck(c, check.Panics, false, "Function must take zero arguments", 1, "BOOM") - - // Plain strings. - testCheck(c, check.Panics, true, "", func() { panic("BOOM") }, "BOOM") - testCheck(c, check.Panics, false, "", func() { panic("KABOOM") }, "BOOM") - testCheck(c, check.Panics, true, "", func() bool { panic("BOOM") }, "BOOM") - - // Error values. - testCheck(c, check.Panics, true, "", func() { panic(errors.New("BOOM")) }, errors.New("BOOM")) - testCheck(c, check.Panics, false, "", func() { panic(errors.New("KABOOM")) }, errors.New("BOOM")) - - type deep struct{ i int } - // Deep value - testCheck(c, check.Panics, true, "", func() { panic(&deep{99}) }, &deep{99}) - - // Verify params/names mutation - params, names := testCheck(c, check.Panics, false, "", func() { panic(errors.New("KABOOM")) }, errors.New("BOOM")) - c.Assert(params[0], check.ErrorMatches, "KABOOM") - c.Assert(names[0], check.Equals, "panic") - - // Verify a nil panic - testCheck(c, check.Panics, true, "", func() { panic(nil) }, nil) - testCheck(c, check.Panics, false, "", func() { panic(nil) }, "NOPE") -} - -func (s *CheckersS) TestPanicMatches(c *check.C) { - testInfo(c, check.PanicMatches, "PanicMatches", []string{"function", "expected"}) - - // Error matching. - testCheck(c, check.PanicMatches, true, "", func() { panic(errors.New("BOOM")) }, "BO.M") - testCheck(c, check.PanicMatches, false, "", func() { panic(errors.New("KABOOM")) }, "BO.M") - - // Some errors. - testCheck(c, check.PanicMatches, false, "Function has not panicked", func() bool { return false }, "BOOM") - testCheck(c, check.PanicMatches, false, "Function must take zero arguments", 1, "BOOM") - - // Plain strings. - testCheck(c, check.PanicMatches, true, "", func() { panic("BOOM") }, "BO.M") - testCheck(c, check.PanicMatches, false, "", func() { panic("KABOOM") }, "BOOM") - testCheck(c, check.PanicMatches, true, "", func() bool { panic("BOOM") }, "BO.M") - - // Verify params/names mutation - params, names := testCheck(c, check.PanicMatches, false, "", func() { panic(errors.New("KABOOM")) }, "BOOM") - c.Assert(params[0], check.Equals, "KABOOM") - c.Assert(names[0], check.Equals, "panic") - - // Verify a nil panic - testCheck(c, check.PanicMatches, false, "Panic value is not a string or an error", func() { panic(nil) }, "") -} - -func (s *CheckersS) TestFitsTypeOf(c *check.C) { - testInfo(c, check.FitsTypeOf, "FitsTypeOf", []string{"obtained", "sample"}) - - // Basic types - testCheck(c, check.FitsTypeOf, true, "", 1, 0) - testCheck(c, check.FitsTypeOf, false, "", 1, int64(0)) - - // Aliases - testCheck(c, check.FitsTypeOf, false, "", 1, errors.New("")) - testCheck(c, check.FitsTypeOf, false, "", "error", errors.New("")) - testCheck(c, check.FitsTypeOf, true, "", errors.New("error"), errors.New("")) - - // Structures - testCheck(c, check.FitsTypeOf, false, "", 1, simpleStruct{}) - testCheck(c, check.FitsTypeOf, false, "", simpleStruct{42}, &simpleStruct{}) - testCheck(c, check.FitsTypeOf, true, "", simpleStruct{42}, simpleStruct{}) - testCheck(c, check.FitsTypeOf, true, "", &simpleStruct{42}, &simpleStruct{}) - - // Some bad values - testCheck(c, check.FitsTypeOf, false, "Invalid sample value", 1, interface{}(nil)) - testCheck(c, check.FitsTypeOf, false, "", interface{}(nil), 0) -} - -func (s *CheckersS) TestImplements(c *check.C) { - testInfo(c, check.Implements, "Implements", []string{"obtained", "ifaceptr"}) - - var e error - var re runtime.Error - testCheck(c, check.Implements, true, "", errors.New(""), &e) - testCheck(c, check.Implements, false, "", errors.New(""), &re) - - // Some bad values - testCheck(c, check.Implements, false, "ifaceptr should be a pointer to an interface variable", 0, errors.New("")) - testCheck(c, check.Implements, false, "ifaceptr should be a pointer to an interface variable", 0, interface{}(nil)) - testCheck(c, check.Implements, false, "", interface{}(nil), &e) -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/export_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/export_test.go deleted file mode 100644 index 0e6cfe0f..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/export_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package check - -func PrintLine(filename string, line int) (string, error) { - return printLine(filename, line) -} - -func Indent(s, with string) string { - return indent(s, with) -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/fixture_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/fixture_test.go deleted file mode 100644 index eaff23a1..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/fixture_test.go +++ /dev/null @@ -1,484 +0,0 @@ -// Tests for the behavior of the test fixture system. - -package check_test - -import ( - . "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" -) - -// ----------------------------------------------------------------------- -// Fixture test suite. - -type FixtureS struct{} - -var fixtureS = Suite(&FixtureS{}) - -func (s *FixtureS) TestCountSuite(c *C) { - suitesRun += 1 -} - -// ----------------------------------------------------------------------- -// Basic fixture ordering verification. - -func (s *FixtureS) TestOrder(c *C) { - helper := FixtureHelper{} - Run(&helper, nil) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "SetUpTest") - c.Check(helper.calls[5], Equals, "Test2") - c.Check(helper.calls[6], Equals, "TearDownTest") - c.Check(helper.calls[7], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 8) -} - -// ----------------------------------------------------------------------- -// Check the behavior when panics occur within tests and fixtures. - -func (s *FixtureS) TestPanicOnTest(c *C) { - helper := FixtureHelper{panicOn: "Test1"} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "SetUpTest") - c.Check(helper.calls[5], Equals, "Test2") - c.Check(helper.calls[6], Equals, "TearDownTest") - c.Check(helper.calls[7], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 8) - - expected := "^\n-+\n" + - "PANIC: check_test\\.go:[0-9]+: FixtureHelper.Test1\n\n" + - "\\.\\.\\. Panic: Test1 \\(PC=[xA-F0-9]+\\)\n\n" + - ".+:[0-9]+\n" + - " in (go)?panic\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.trace\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.Test1\n" + - "(.|\n)*$" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnSetUpTest(c *C) { - helper := FixtureHelper{panicOn: "SetUpTest"} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "TearDownTest") - c.Check(helper.calls[3], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 4) - - expected := "^\n-+\n" + - "PANIC: check_test\\.go:[0-9]+: " + - "FixtureHelper\\.SetUpTest\n\n" + - "\\.\\.\\. Panic: SetUpTest \\(PC=[xA-F0-9]+\\)\n\n" + - ".+:[0-9]+\n" + - " in (go)?panic\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.trace\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.SetUpTest\n" + - "(.|\n)*" + - "\n-+\n" + - "PANIC: check_test\\.go:[0-9]+: " + - "FixtureHelper\\.Test1\n\n" + - "\\.\\.\\. Panic: Fixture has panicked " + - "\\(see related PANIC\\)\n$" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnTearDownTest(c *C) { - helper := FixtureHelper{panicOn: "TearDownTest"} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 5) - - expected := "^\n-+\n" + - "PANIC: check_test\\.go:[0-9]+: " + - "FixtureHelper.TearDownTest\n\n" + - "\\.\\.\\. Panic: TearDownTest \\(PC=[xA-F0-9]+\\)\n\n" + - ".+:[0-9]+\n" + - " in (go)?panic\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.trace\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.TearDownTest\n" + - "(.|\n)*" + - "\n-+\n" + - "PANIC: check_test\\.go:[0-9]+: " + - "FixtureHelper\\.Test1\n\n" + - "\\.\\.\\. Panic: Fixture has panicked " + - "\\(see related PANIC\\)\n$" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnSetUpSuite(c *C) { - helper := FixtureHelper{panicOn: "SetUpSuite"} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 2) - - expected := "^\n-+\n" + - "PANIC: check_test\\.go:[0-9]+: " + - "FixtureHelper.SetUpSuite\n\n" + - "\\.\\.\\. Panic: SetUpSuite \\(PC=[xA-F0-9]+\\)\n\n" + - ".+:[0-9]+\n" + - " in (go)?panic\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.trace\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.SetUpSuite\n" + - "(.|\n)*$" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnTearDownSuite(c *C) { - helper := FixtureHelper{panicOn: "TearDownSuite"} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "SetUpTest") - c.Check(helper.calls[5], Equals, "Test2") - c.Check(helper.calls[6], Equals, "TearDownTest") - c.Check(helper.calls[7], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 8) - - expected := "^\n-+\n" + - "PANIC: check_test\\.go:[0-9]+: " + - "FixtureHelper.TearDownSuite\n\n" + - "\\.\\.\\. Panic: TearDownSuite \\(PC=[xA-F0-9]+\\)\n\n" + - ".+:[0-9]+\n" + - " in (go)?panic\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.trace\n" + - ".*check_test.go:[0-9]+\n" + - " in FixtureHelper.TearDownSuite\n" + - "(.|\n)*$" - - c.Check(output.value, Matches, expected) -} - -// ----------------------------------------------------------------------- -// A wrong argument on a test or fixture will produce a nice error. - -func (s *FixtureS) TestPanicOnWrongTestArg(c *C) { - helper := WrongTestArgHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "TearDownTest") - c.Check(helper.calls[3], Equals, "SetUpTest") - c.Check(helper.calls[4], Equals, "Test2") - c.Check(helper.calls[5], Equals, "TearDownTest") - c.Check(helper.calls[6], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 7) - - expected := "^\n-+\n" + - "PANIC: fixture_test\\.go:[0-9]+: " + - "WrongTestArgHelper\\.Test1\n\n" + - "\\.\\.\\. Panic: WrongTestArgHelper\\.Test1 argument " + - "should be \\*check\\.C\n" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnWrongSetUpTestArg(c *C) { - helper := WrongSetUpTestArgHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(len(helper.calls), Equals, 0) - - expected := - "^\n-+\n" + - "PANIC: fixture_test\\.go:[0-9]+: " + - "WrongSetUpTestArgHelper\\.SetUpTest\n\n" + - "\\.\\.\\. Panic: WrongSetUpTestArgHelper\\.SetUpTest argument " + - "should be \\*check\\.C\n" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnWrongSetUpSuiteArg(c *C) { - helper := WrongSetUpSuiteArgHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(len(helper.calls), Equals, 0) - - expected := - "^\n-+\n" + - "PANIC: fixture_test\\.go:[0-9]+: " + - "WrongSetUpSuiteArgHelper\\.SetUpSuite\n\n" + - "\\.\\.\\. Panic: WrongSetUpSuiteArgHelper\\.SetUpSuite argument " + - "should be \\*check\\.C\n" - - c.Check(output.value, Matches, expected) -} - -// ----------------------------------------------------------------------- -// Nice errors also when tests or fixture have wrong arg count. - -func (s *FixtureS) TestPanicOnWrongTestArgCount(c *C) { - helper := WrongTestArgCountHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "TearDownTest") - c.Check(helper.calls[3], Equals, "SetUpTest") - c.Check(helper.calls[4], Equals, "Test2") - c.Check(helper.calls[5], Equals, "TearDownTest") - c.Check(helper.calls[6], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 7) - - expected := "^\n-+\n" + - "PANIC: fixture_test\\.go:[0-9]+: " + - "WrongTestArgCountHelper\\.Test1\n\n" + - "\\.\\.\\. Panic: WrongTestArgCountHelper\\.Test1 argument " + - "should be \\*check\\.C\n" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnWrongSetUpTestArgCount(c *C) { - helper := WrongSetUpTestArgCountHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(len(helper.calls), Equals, 0) - - expected := - "^\n-+\n" + - "PANIC: fixture_test\\.go:[0-9]+: " + - "WrongSetUpTestArgCountHelper\\.SetUpTest\n\n" + - "\\.\\.\\. Panic: WrongSetUpTestArgCountHelper\\.SetUpTest argument " + - "should be \\*check\\.C\n" - - c.Check(output.value, Matches, expected) -} - -func (s *FixtureS) TestPanicOnWrongSetUpSuiteArgCount(c *C) { - helper := WrongSetUpSuiteArgCountHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(len(helper.calls), Equals, 0) - - expected := - "^\n-+\n" + - "PANIC: fixture_test\\.go:[0-9]+: " + - "WrongSetUpSuiteArgCountHelper\\.SetUpSuite\n\n" + - "\\.\\.\\. Panic: WrongSetUpSuiteArgCountHelper" + - "\\.SetUpSuite argument should be \\*check\\.C\n" - - c.Check(output.value, Matches, expected) -} - -// ----------------------------------------------------------------------- -// Helper test suites with wrong function arguments. - -type WrongTestArgHelper struct { - FixtureHelper -} - -func (s *WrongTestArgHelper) Test1(t int) { -} - -type WrongSetUpTestArgHelper struct { - FixtureHelper -} - -func (s *WrongSetUpTestArgHelper) SetUpTest(t int) { -} - -type WrongSetUpSuiteArgHelper struct { - FixtureHelper -} - -func (s *WrongSetUpSuiteArgHelper) SetUpSuite(t int) { -} - -type WrongTestArgCountHelper struct { - FixtureHelper -} - -func (s *WrongTestArgCountHelper) Test1(c *C, i int) { -} - -type WrongSetUpTestArgCountHelper struct { - FixtureHelper -} - -func (s *WrongSetUpTestArgCountHelper) SetUpTest(c *C, i int) { -} - -type WrongSetUpSuiteArgCountHelper struct { - FixtureHelper -} - -func (s *WrongSetUpSuiteArgCountHelper) SetUpSuite(c *C, i int) { -} - -// ----------------------------------------------------------------------- -// Ensure fixture doesn't run without tests. - -type NoTestsHelper struct { - hasRun bool -} - -func (s *NoTestsHelper) SetUpSuite(c *C) { - s.hasRun = true -} - -func (s *NoTestsHelper) TearDownSuite(c *C) { - s.hasRun = true -} - -func (s *FixtureS) TestFixtureDoesntRunWithoutTests(c *C) { - helper := NoTestsHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Check(helper.hasRun, Equals, false) -} - -// ----------------------------------------------------------------------- -// Verify that checks and assertions work correctly inside the fixture. - -type FixtureCheckHelper struct { - fail string - completed bool -} - -func (s *FixtureCheckHelper) SetUpSuite(c *C) { - switch s.fail { - case "SetUpSuiteAssert": - c.Assert(false, Equals, true) - case "SetUpSuiteCheck": - c.Check(false, Equals, true) - } - s.completed = true -} - -func (s *FixtureCheckHelper) SetUpTest(c *C) { - switch s.fail { - case "SetUpTestAssert": - c.Assert(false, Equals, true) - case "SetUpTestCheck": - c.Check(false, Equals, true) - } - s.completed = true -} - -func (s *FixtureCheckHelper) Test(c *C) { - // Do nothing. -} - -func (s *FixtureS) TestSetUpSuiteCheck(c *C) { - helper := FixtureCheckHelper{fail: "SetUpSuiteCheck"} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Assert(output.value, Matches, - "\n---+\n"+ - "FAIL: fixture_test\\.go:[0-9]+: "+ - "FixtureCheckHelper\\.SetUpSuite\n\n"+ - "fixture_test\\.go:[0-9]+:\n"+ - " c\\.Check\\(false, Equals, true\\)\n"+ - "\\.+ obtained bool = false\n"+ - "\\.+ expected bool = true\n\n") - c.Assert(helper.completed, Equals, true) -} - -func (s *FixtureS) TestSetUpSuiteAssert(c *C) { - helper := FixtureCheckHelper{fail: "SetUpSuiteAssert"} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Assert(output.value, Matches, - "\n---+\n"+ - "FAIL: fixture_test\\.go:[0-9]+: "+ - "FixtureCheckHelper\\.SetUpSuite\n\n"+ - "fixture_test\\.go:[0-9]+:\n"+ - " c\\.Assert\\(false, Equals, true\\)\n"+ - "\\.+ obtained bool = false\n"+ - "\\.+ expected bool = true\n\n") - c.Assert(helper.completed, Equals, false) -} - -// ----------------------------------------------------------------------- -// Verify that logging within SetUpTest() persists within the test log itself. - -type FixtureLogHelper struct { - c *C -} - -func (s *FixtureLogHelper) SetUpTest(c *C) { - s.c = c - c.Log("1") -} - -func (s *FixtureLogHelper) Test(c *C) { - c.Log("2") - s.c.Log("3") - c.Log("4") - c.Fail() -} - -func (s *FixtureLogHelper) TearDownTest(c *C) { - s.c.Log("5") -} - -func (s *FixtureS) TestFixtureLogging(c *C) { - helper := FixtureLogHelper{} - output := String{} - Run(&helper, &RunConf{Output: &output}) - c.Assert(output.value, Matches, - "\n---+\n"+ - "FAIL: fixture_test\\.go:[0-9]+: "+ - "FixtureLogHelper\\.Test\n\n"+ - "1\n2\n3\n4\n5\n") -} - -// ----------------------------------------------------------------------- -// Skip() within fixture methods. - -func (s *FixtureS) TestSkipSuite(c *C) { - helper := FixtureHelper{skip: true, skipOnN: 0} - output := String{} - result := Run(&helper, &RunConf{Output: &output}) - c.Assert(output.value, Equals, "") - c.Assert(helper.calls[0], Equals, "SetUpSuite") - c.Assert(helper.calls[1], Equals, "TearDownSuite") - c.Assert(len(helper.calls), Equals, 2) - c.Assert(result.Skipped, Equals, 2) -} - -func (s *FixtureS) TestSkipTest(c *C) { - helper := FixtureHelper{skip: true, skipOnN: 1} - output := String{} - result := Run(&helper, &RunConf{Output: &output}) - c.Assert(helper.calls[0], Equals, "SetUpSuite") - c.Assert(helper.calls[1], Equals, "SetUpTest") - c.Assert(helper.calls[2], Equals, "SetUpTest") - c.Assert(helper.calls[3], Equals, "Test2") - c.Assert(helper.calls[4], Equals, "TearDownTest") - c.Assert(helper.calls[5], Equals, "TearDownSuite") - c.Assert(len(helper.calls), Equals, 6) - c.Assert(result.Skipped, Equals, 1) -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/foundation_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/foundation_test.go deleted file mode 100644 index 809ef65e..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/foundation_test.go +++ /dev/null @@ -1,335 +0,0 @@ -// These tests check that the foundations of gocheck are working properly. -// They already assume that fundamental failing is working already, though, -// since this was tested in bootstrap_test.go. Even then, some care may -// still have to be taken when using external functions, since they should -// of course not rely on functionality tested here. - -package check_test - -import ( - "fmt" - "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" - "log" - "os" - "regexp" - "strings" -) - -// ----------------------------------------------------------------------- -// Foundation test suite. - -type FoundationS struct{} - -var foundationS = check.Suite(&FoundationS{}) - -func (s *FoundationS) TestCountSuite(c *check.C) { - suitesRun += 1 -} - -func (s *FoundationS) TestErrorf(c *check.C) { - // Do not use checkState() here. It depends on Errorf() working. - expectedLog := fmt.Sprintf("foundation_test.go:%d:\n"+ - " c.Errorf(\"Error %%v!\", \"message\")\n"+ - "... Error: Error message!\n\n", - getMyLine()+1) - c.Errorf("Error %v!", "message") - failed := c.Failed() - c.Succeed() - if log := c.GetTestLog(); log != expectedLog { - c.Logf("Errorf() logged %#v rather than %#v", log, expectedLog) - c.Fail() - } - if !failed { - c.Logf("Errorf() didn't put the test in a failed state") - c.Fail() - } -} - -func (s *FoundationS) TestError(c *check.C) { - expectedLog := fmt.Sprintf("foundation_test.go:%d:\n"+ - " c\\.Error\\(\"Error \", \"message!\"\\)\n"+ - "\\.\\.\\. Error: Error message!\n\n", - getMyLine()+1) - c.Error("Error ", "message!") - checkState(c, nil, - &expectedState{ - name: "Error(`Error `, `message!`)", - failed: true, - log: expectedLog, - }) -} - -func (s *FoundationS) TestFailNow(c *check.C) { - defer (func() { - if !c.Failed() { - c.Error("FailNow() didn't fail the test") - } else { - c.Succeed() - if c.GetTestLog() != "" { - c.Error("Something got logged:\n" + c.GetTestLog()) - } - } - })() - - c.FailNow() - c.Log("FailNow() didn't stop the test") -} - -func (s *FoundationS) TestSucceedNow(c *check.C) { - defer (func() { - if c.Failed() { - c.Error("SucceedNow() didn't succeed the test") - } - if c.GetTestLog() != "" { - c.Error("Something got logged:\n" + c.GetTestLog()) - } - })() - - c.Fail() - c.SucceedNow() - c.Log("SucceedNow() didn't stop the test") -} - -func (s *FoundationS) TestFailureHeader(c *check.C) { - output := String{} - failHelper := FailHelper{} - check.Run(&failHelper, &check.RunConf{Output: &output}) - header := fmt.Sprintf(""+ - "\n-----------------------------------"+ - "-----------------------------------\n"+ - "FAIL: check_test.go:%d: FailHelper.TestLogAndFail\n", - failHelper.testLine) - if strings.Index(output.value, header) == -1 { - c.Errorf(""+ - "Failure didn't print a proper header.\n"+ - "... Got:\n%s... Expected something with:\n%s", - output.value, header) - } -} - -func (s *FoundationS) TestFatal(c *check.C) { - var line int - defer (func() { - if !c.Failed() { - c.Error("Fatal() didn't fail the test") - } else { - c.Succeed() - expected := fmt.Sprintf("foundation_test.go:%d:\n"+ - " c.Fatal(\"Die \", \"now!\")\n"+ - "... Error: Die now!\n\n", - line) - if c.GetTestLog() != expected { - c.Error("Incorrect log:", c.GetTestLog()) - } - } - })() - - line = getMyLine() + 1 - c.Fatal("Die ", "now!") - c.Log("Fatal() didn't stop the test") -} - -func (s *FoundationS) TestFatalf(c *check.C) { - var line int - defer (func() { - if !c.Failed() { - c.Error("Fatalf() didn't fail the test") - } else { - c.Succeed() - expected := fmt.Sprintf("foundation_test.go:%d:\n"+ - " c.Fatalf(\"Die %%s!\", \"now\")\n"+ - "... Error: Die now!\n\n", - line) - if c.GetTestLog() != expected { - c.Error("Incorrect log:", c.GetTestLog()) - } - } - })() - - line = getMyLine() + 1 - c.Fatalf("Die %s!", "now") - c.Log("Fatalf() didn't stop the test") -} - -func (s *FoundationS) TestCallerLoggingInsideTest(c *check.C) { - log := fmt.Sprintf(""+ - "foundation_test.go:%d:\n"+ - " result := c.Check\\(10, check.Equals, 20\\)\n"+ - "\\.\\.\\. obtained int = 10\n"+ - "\\.\\.\\. expected int = 20\n\n", - getMyLine()+1) - result := c.Check(10, check.Equals, 20) - checkState(c, result, - &expectedState{ - name: "Check(10, Equals, 20)", - result: false, - failed: true, - log: log, - }) -} - -func (s *FoundationS) TestCallerLoggingInDifferentFile(c *check.C) { - result, line := checkEqualWrapper(c, 10, 20) - testLine := getMyLine() - 1 - log := fmt.Sprintf(""+ - "foundation_test.go:%d:\n"+ - " result, line := checkEqualWrapper\\(c, 10, 20\\)\n"+ - "check_test.go:%d:\n"+ - " return c.Check\\(obtained, check.Equals, expected\\), getMyLine\\(\\)\n"+ - "\\.\\.\\. obtained int = 10\n"+ - "\\.\\.\\. expected int = 20\n\n", - testLine, line) - checkState(c, result, - &expectedState{ - name: "Check(10, Equals, 20)", - result: false, - failed: true, - log: log, - }) -} - -// ----------------------------------------------------------------------- -// ExpectFailure() inverts the logic of failure. - -type ExpectFailureSucceedHelper struct{} - -func (s *ExpectFailureSucceedHelper) TestSucceed(c *check.C) { - c.ExpectFailure("It booms!") - c.Error("Boom!") -} - -type ExpectFailureFailHelper struct{} - -func (s *ExpectFailureFailHelper) TestFail(c *check.C) { - c.ExpectFailure("Bug #XYZ") -} - -func (s *FoundationS) TestExpectFailureFail(c *check.C) { - helper := ExpectFailureFailHelper{} - output := String{} - result := check.Run(&helper, &check.RunConf{Output: &output}) - - expected := "" + - "^\n-+\n" + - "FAIL: foundation_test\\.go:[0-9]+:" + - " ExpectFailureFailHelper\\.TestFail\n\n" + - "\\.\\.\\. Error: Test succeeded, but was expected to fail\n" + - "\\.\\.\\. Reason: Bug #XYZ\n$" - - matched, err := regexp.MatchString(expected, output.value) - if err != nil { - c.Error("Bad expression: ", expected) - } else if !matched { - c.Error("ExpectFailure() didn't log properly:\n", output.value) - } - - c.Assert(result.ExpectedFailures, check.Equals, 0) -} - -func (s *FoundationS) TestExpectFailureSucceed(c *check.C) { - helper := ExpectFailureSucceedHelper{} - output := String{} - result := check.Run(&helper, &check.RunConf{Output: &output}) - - c.Assert(output.value, check.Equals, "") - c.Assert(result.ExpectedFailures, check.Equals, 1) -} - -func (s *FoundationS) TestExpectFailureSucceedVerbose(c *check.C) { - helper := ExpectFailureSucceedHelper{} - output := String{} - result := check.Run(&helper, &check.RunConf{Output: &output, Verbose: true}) - - expected := "" + - "FAIL EXPECTED: foundation_test\\.go:[0-9]+:" + - " ExpectFailureSucceedHelper\\.TestSucceed \\(It booms!\\)\t *[.0-9]+s\n" - - matched, err := regexp.MatchString(expected, output.value) - if err != nil { - c.Error("Bad expression: ", expected) - } else if !matched { - c.Error("ExpectFailure() didn't log properly:\n", output.value) - } - - c.Assert(result.ExpectedFailures, check.Equals, 1) -} - -// ----------------------------------------------------------------------- -// Skip() allows stopping a test without positive/negative results. - -type SkipTestHelper struct{} - -func (s *SkipTestHelper) TestFail(c *check.C) { - c.Skip("Wrong platform or whatever") - c.Error("Boom!") -} - -func (s *FoundationS) TestSkip(c *check.C) { - helper := SkipTestHelper{} - output := String{} - check.Run(&helper, &check.RunConf{Output: &output}) - - if output.value != "" { - c.Error("Skip() logged something:\n", output.value) - } -} - -func (s *FoundationS) TestSkipVerbose(c *check.C) { - helper := SkipTestHelper{} - output := String{} - check.Run(&helper, &check.RunConf{Output: &output, Verbose: true}) - - expected := "SKIP: foundation_test\\.go:[0-9]+: SkipTestHelper\\.TestFail" + - " \\(Wrong platform or whatever\\)" - matched, err := regexp.MatchString(expected, output.value) - if err != nil { - c.Error("Bad expression: ", expected) - } else if !matched { - c.Error("Skip() didn't log properly:\n", output.value) - } -} - -// ----------------------------------------------------------------------- -// Check minimum *log.Logger interface provided by *check.C. - -type minLogger interface { - Output(calldepth int, s string) error -} - -func (s *BootstrapS) TestMinLogger(c *check.C) { - var logger minLogger - logger = log.New(os.Stderr, "", 0) - logger = c - logger.Output(0, "Hello there") - expected := `\[LOG\] [0-9]+:[0-9][0-9]\.[0-9][0-9][0-9] +Hello there\n` - output := c.GetTestLog() - c.Assert(output, check.Matches, expected) -} - -// ----------------------------------------------------------------------- -// Ensure that suites with embedded types are working fine, including the -// the workaround for issue 906. - -type EmbeddedInternalS struct { - called bool -} - -type EmbeddedS struct { - EmbeddedInternalS -} - -var embeddedS = check.Suite(&EmbeddedS{}) - -func (s *EmbeddedS) TestCountSuite(c *check.C) { - suitesRun += 1 -} - -func (s *EmbeddedInternalS) TestMethod(c *check.C) { - c.Error("TestMethod() of the embedded type was called!?") -} - -func (s *EmbeddedS) TestMethod(c *check.C) { - // http://code.google.com/p/go/issues/detail?id=906 - c.Check(s.called, check.Equals, false) // Go issue 906 is affecting the runner? - s.called = true -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers.go deleted file mode 100644 index 4b6c26da..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers.go +++ /dev/null @@ -1,231 +0,0 @@ -package check - -import ( - "fmt" - "strings" - "time" -) - -// TestName returns the current test name in the form "SuiteName.TestName" -func (c *C) TestName() string { - return c.testName -} - -// ----------------------------------------------------------------------- -// Basic succeeding/failing logic. - -// Failed returns whether the currently running test has already failed. -func (c *C) Failed() bool { - return c.status == failedSt -} - -// Fail marks the currently running test as failed. -// -// Something ought to have been previously logged so the developer can tell -// what went wrong. The higher level helper functions will fail the test -// and do the logging properly. -func (c *C) Fail() { - c.status = failedSt -} - -// FailNow marks the currently running test as failed and stops running it. -// Something ought to have been previously logged so the developer can tell -// what went wrong. The higher level helper functions will fail the test -// and do the logging properly. -func (c *C) FailNow() { - c.Fail() - c.stopNow() -} - -// Succeed marks the currently running test as succeeded, undoing any -// previous failures. -func (c *C) Succeed() { - c.status = succeededSt -} - -// SucceedNow marks the currently running test as succeeded, undoing any -// previous failures, and stops running the test. -func (c *C) SucceedNow() { - c.Succeed() - c.stopNow() -} - -// ExpectFailure informs that the running test is knowingly broken for -// the provided reason. If the test does not fail, an error will be reported -// to raise attention to this fact. This method is useful to temporarily -// disable tests which cover well known problems until a better time to -// fix the problem is found, without forgetting about the fact that a -// failure still exists. -func (c *C) ExpectFailure(reason string) { - if reason == "" { - panic("Missing reason why the test is expected to fail") - } - c.mustFail = true - c.reason = reason -} - -// Skip skips the running test for the provided reason. If run from within -// SetUpTest, the individual test being set up will be skipped, and if run -// from within SetUpSuite, the whole suite is skipped. -func (c *C) Skip(reason string) { - if reason == "" { - panic("Missing reason why the test is being skipped") - } - c.reason = reason - c.status = skippedSt - c.stopNow() -} - -// ----------------------------------------------------------------------- -// Basic logging. - -// GetTestLog returns the current test error output. -func (c *C) GetTestLog() string { - return c.logb.String() -} - -// Log logs some information into the test error output. -// The provided arguments are assembled together into a string with fmt.Sprint. -func (c *C) Log(args ...interface{}) { - c.log(args...) -} - -// Log logs some information into the test error output. -// The provided arguments are assembled together into a string with fmt.Sprintf. -func (c *C) Logf(format string, args ...interface{}) { - c.logf(format, args...) -} - -// Output enables *C to be used as a logger in functions that require only -// the minimum interface of *log.Logger. -func (c *C) Output(calldepth int, s string) error { - d := time.Now().Sub(c.startTime) - msec := d / time.Millisecond - sec := d / time.Second - min := d / time.Minute - - c.Logf("[LOG] %d:%02d.%03d %s", min, sec%60, msec%1000, s) - return nil -} - -// Error logs an error into the test error output and marks the test as failed. -// The provided arguments are assembled together into a string with fmt.Sprint. -func (c *C) Error(args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) - c.logNewLine() - c.Fail() -} - -// Errorf logs an error into the test error output and marks the test as failed. -// The provided arguments are assembled together into a string with fmt.Sprintf. -func (c *C) Errorf(format string, args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprintf("Error: "+format, args...)) - c.logNewLine() - c.Fail() -} - -// Fatal logs an error into the test error output, marks the test as failed, and -// stops the test execution. The provided arguments are assembled together into -// a string with fmt.Sprint. -func (c *C) Fatal(args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprint("Error: ", fmt.Sprint(args...))) - c.logNewLine() - c.FailNow() -} - -// Fatlaf logs an error into the test error output, marks the test as failed, and -// stops the test execution. The provided arguments are assembled together into -// a string with fmt.Sprintf. -func (c *C) Fatalf(format string, args ...interface{}) { - c.logCaller(1) - c.logString(fmt.Sprint("Error: ", fmt.Sprintf(format, args...))) - c.logNewLine() - c.FailNow() -} - -// ----------------------------------------------------------------------- -// Generic checks and assertions based on checkers. - -// Check verifies if the first value matches the expected value according -// to the provided checker. If they do not match, an error is logged, the -// test is marked as failed, and the test execution continues. -// -// Some checkers may not need the expected argument (e.g. IsNil). -// -// Extra arguments provided to the function are logged next to the reported -// problem when the matching fails. -func (c *C) Check(obtained interface{}, checker Checker, args ...interface{}) bool { - return c.internalCheck("Check", obtained, checker, args...) -} - -// Assert ensures that the first value matches the expected value according -// to the provided checker. If they do not match, an error is logged, the -// test is marked as failed, and the test execution stops. -// -// Some checkers may not need the expected argument (e.g. IsNil). -// -// Extra arguments provided to the function are logged next to the reported -// problem when the matching fails. -func (c *C) Assert(obtained interface{}, checker Checker, args ...interface{}) { - if !c.internalCheck("Assert", obtained, checker, args...) { - c.stopNow() - } -} - -func (c *C) internalCheck(funcName string, obtained interface{}, checker Checker, args ...interface{}) bool { - if checker == nil { - c.logCaller(2) - c.logString(fmt.Sprintf("%s(obtained, nil!?, ...):", funcName)) - c.logString("Oops.. you've provided a nil checker!") - c.logNewLine() - c.Fail() - return false - } - - // If the last argument is a bug info, extract it out. - var comment CommentInterface - if len(args) > 0 { - if c, ok := args[len(args)-1].(CommentInterface); ok { - comment = c - args = args[:len(args)-1] - } - } - - params := append([]interface{}{obtained}, args...) - info := checker.Info() - - if len(params) != len(info.Params) { - names := append([]string{info.Params[0], info.Name}, info.Params[1:]...) - c.logCaller(2) - c.logString(fmt.Sprintf("%s(%s):", funcName, strings.Join(names, ", "))) - c.logString(fmt.Sprintf("Wrong number of parameters for %s: want %d, got %d", info.Name, len(names), len(params)+1)) - c.logNewLine() - c.Fail() - return false - } - - // Copy since it may be mutated by Check. - names := append([]string{}, info.Params...) - - // Do the actual check. - result, error := checker.Check(params, names) - if !result || error != "" { - c.logCaller(2) - for i := 0; i != len(params); i++ { - c.logValue(names[i], params[i]) - } - if comment != nil { - c.logString(comment.CheckCommentString()) - } - if error != "" { - c.logString(error) - } - c.logNewLine() - c.Fail() - return false - } - return true -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers_test.go deleted file mode 100644 index 704ee101..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/helpers_test.go +++ /dev/null @@ -1,519 +0,0 @@ -// These tests verify the inner workings of the helper methods associated -// with check.T. - -package check_test - -import ( - "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" - "os" - "reflect" - "runtime" - "sync" -) - -var helpersS = check.Suite(&HelpersS{}) - -type HelpersS struct{} - -func (s *HelpersS) TestCountSuite(c *check.C) { - suitesRun += 1 -} - -// ----------------------------------------------------------------------- -// Fake checker and bug info to verify the behavior of Assert() and Check(). - -type MyChecker struct { - info *check.CheckerInfo - params []interface{} - names []string - result bool - error string -} - -func (checker *MyChecker) Info() *check.CheckerInfo { - if checker.info == nil { - return &check.CheckerInfo{Name: "MyChecker", Params: []string{"myobtained", "myexpected"}} - } - return checker.info -} - -func (checker *MyChecker) Check(params []interface{}, names []string) (bool, string) { - rparams := checker.params - rnames := checker.names - checker.params = append([]interface{}{}, params...) - checker.names = append([]string{}, names...) - if rparams != nil { - copy(params, rparams) - } - if rnames != nil { - copy(names, rnames) - } - return checker.result, checker.error -} - -type myCommentType string - -func (c myCommentType) CheckCommentString() string { - return string(c) -} - -func myComment(s string) myCommentType { - return myCommentType(s) -} - -// ----------------------------------------------------------------------- -// Ensure a real checker actually works fine. - -func (s *HelpersS) TestCheckerInterface(c *check.C) { - testHelperSuccess(c, "Check(1, Equals, 1)", true, func() interface{} { - return c.Check(1, check.Equals, 1) - }) -} - -// ----------------------------------------------------------------------- -// Tests for Check(), mostly the same as for Assert() following these. - -func (s *HelpersS) TestCheckSucceedWithExpected(c *check.C) { - checker := &MyChecker{result: true} - testHelperSuccess(c, "Check(1, checker, 2)", true, func() interface{} { - return c.Check(1, checker, 2) - }) - if !reflect.DeepEqual(checker.params, []interface{}{1, 2}) { - c.Fatalf("Bad params for check: %#v", checker.params) - } -} - -func (s *HelpersS) TestCheckSucceedWithoutExpected(c *check.C) { - checker := &MyChecker{result: true, info: &check.CheckerInfo{Params: []string{"myvalue"}}} - testHelperSuccess(c, "Check(1, checker)", true, func() interface{} { - return c.Check(1, checker) - }) - if !reflect.DeepEqual(checker.params, []interface{}{1}) { - c.Fatalf("Bad params for check: %#v", checker.params) - } -} - -func (s *HelpersS) TestCheckFailWithExpected(c *check.C) { - checker := &MyChecker{result: false} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker, 2\\)\n" + - "\\.+ myobtained int = 1\n" + - "\\.+ myexpected int = 2\n\n" - testHelperFailure(c, "Check(1, checker, 2)", false, false, log, - func() interface{} { - return c.Check(1, checker, 2) - }) -} - -func (s *HelpersS) TestCheckFailWithExpectedAndComment(c *check.C) { - checker := &MyChecker{result: false} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker, 2, myComment\\(\"Hello world!\"\\)\\)\n" + - "\\.+ myobtained int = 1\n" + - "\\.+ myexpected int = 2\n" + - "\\.+ Hello world!\n\n" - testHelperFailure(c, "Check(1, checker, 2, msg)", false, false, log, - func() interface{} { - return c.Check(1, checker, 2, myComment("Hello world!")) - }) -} - -func (s *HelpersS) TestCheckFailWithExpectedAndStaticComment(c *check.C) { - checker := &MyChecker{result: false} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " // Nice leading comment\\.\n" + - " return c\\.Check\\(1, checker, 2\\) // Hello there\n" + - "\\.+ myobtained int = 1\n" + - "\\.+ myexpected int = 2\n\n" - testHelperFailure(c, "Check(1, checker, 2, msg)", false, false, log, - func() interface{} { - // Nice leading comment. - return c.Check(1, checker, 2) // Hello there - }) -} - -func (s *HelpersS) TestCheckFailWithoutExpected(c *check.C) { - checker := &MyChecker{result: false, info: &check.CheckerInfo{Params: []string{"myvalue"}}} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker\\)\n" + - "\\.+ myvalue int = 1\n\n" - testHelperFailure(c, "Check(1, checker)", false, false, log, - func() interface{} { - return c.Check(1, checker) - }) -} - -func (s *HelpersS) TestCheckFailWithoutExpectedAndMessage(c *check.C) { - checker := &MyChecker{result: false, info: &check.CheckerInfo{Params: []string{"myvalue"}}} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker, myComment\\(\"Hello world!\"\\)\\)\n" + - "\\.+ myvalue int = 1\n" + - "\\.+ Hello world!\n\n" - testHelperFailure(c, "Check(1, checker, msg)", false, false, log, - func() interface{} { - return c.Check(1, checker, myComment("Hello world!")) - }) -} - -func (s *HelpersS) TestCheckWithMissingExpected(c *check.C) { - checker := &MyChecker{result: true} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker\\)\n" + - "\\.+ Check\\(myobtained, MyChecker, myexpected\\):\n" + - "\\.+ Wrong number of parameters for MyChecker: " + - "want 3, got 2\n\n" - testHelperFailure(c, "Check(1, checker, !?)", false, false, log, - func() interface{} { - return c.Check(1, checker) - }) -} - -func (s *HelpersS) TestCheckWithTooManyExpected(c *check.C) { - checker := &MyChecker{result: true} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker, 2, 3\\)\n" + - "\\.+ Check\\(myobtained, MyChecker, myexpected\\):\n" + - "\\.+ Wrong number of parameters for MyChecker: " + - "want 3, got 4\n\n" - testHelperFailure(c, "Check(1, checker, 2, 3)", false, false, log, - func() interface{} { - return c.Check(1, checker, 2, 3) - }) -} - -func (s *HelpersS) TestCheckWithError(c *check.C) { - checker := &MyChecker{result: false, error: "Some not so cool data provided!"} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker, 2\\)\n" + - "\\.+ myobtained int = 1\n" + - "\\.+ myexpected int = 2\n" + - "\\.+ Some not so cool data provided!\n\n" - testHelperFailure(c, "Check(1, checker, 2)", false, false, log, - func() interface{} { - return c.Check(1, checker, 2) - }) -} - -func (s *HelpersS) TestCheckWithNilChecker(c *check.C) { - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, nil\\)\n" + - "\\.+ Check\\(obtained, nil!\\?, \\.\\.\\.\\):\n" + - "\\.+ Oops\\.\\. you've provided a nil checker!\n\n" - testHelperFailure(c, "Check(obtained, nil)", false, false, log, - func() interface{} { - return c.Check(1, nil) - }) -} - -func (s *HelpersS) TestCheckWithParamsAndNamesMutation(c *check.C) { - checker := &MyChecker{result: false, params: []interface{}{3, 4}, names: []string{"newobtained", "newexpected"}} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " return c\\.Check\\(1, checker, 2\\)\n" + - "\\.+ newobtained int = 3\n" + - "\\.+ newexpected int = 4\n\n" - testHelperFailure(c, "Check(1, checker, 2) with mutation", false, false, log, - func() interface{} { - return c.Check(1, checker, 2) - }) -} - -// ----------------------------------------------------------------------- -// Tests for Assert(), mostly the same as for Check() above. - -func (s *HelpersS) TestAssertSucceedWithExpected(c *check.C) { - checker := &MyChecker{result: true} - testHelperSuccess(c, "Assert(1, checker, 2)", nil, func() interface{} { - c.Assert(1, checker, 2) - return nil - }) - if !reflect.DeepEqual(checker.params, []interface{}{1, 2}) { - c.Fatalf("Bad params for check: %#v", checker.params) - } -} - -func (s *HelpersS) TestAssertSucceedWithoutExpected(c *check.C) { - checker := &MyChecker{result: true, info: &check.CheckerInfo{Params: []string{"myvalue"}}} - testHelperSuccess(c, "Assert(1, checker)", nil, func() interface{} { - c.Assert(1, checker) - return nil - }) - if !reflect.DeepEqual(checker.params, []interface{}{1}) { - c.Fatalf("Bad params for check: %#v", checker.params) - } -} - -func (s *HelpersS) TestAssertFailWithExpected(c *check.C) { - checker := &MyChecker{result: false} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " c\\.Assert\\(1, checker, 2\\)\n" + - "\\.+ myobtained int = 1\n" + - "\\.+ myexpected int = 2\n\n" - testHelperFailure(c, "Assert(1, checker, 2)", nil, true, log, - func() interface{} { - c.Assert(1, checker, 2) - return nil - }) -} - -func (s *HelpersS) TestAssertFailWithExpectedAndMessage(c *check.C) { - checker := &MyChecker{result: false} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " c\\.Assert\\(1, checker, 2, myComment\\(\"Hello world!\"\\)\\)\n" + - "\\.+ myobtained int = 1\n" + - "\\.+ myexpected int = 2\n" + - "\\.+ Hello world!\n\n" - testHelperFailure(c, "Assert(1, checker, 2, msg)", nil, true, log, - func() interface{} { - c.Assert(1, checker, 2, myComment("Hello world!")) - return nil - }) -} - -func (s *HelpersS) TestAssertFailWithoutExpected(c *check.C) { - checker := &MyChecker{result: false, info: &check.CheckerInfo{Params: []string{"myvalue"}}} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " c\\.Assert\\(1, checker\\)\n" + - "\\.+ myvalue int = 1\n\n" - testHelperFailure(c, "Assert(1, checker)", nil, true, log, - func() interface{} { - c.Assert(1, checker) - return nil - }) -} - -func (s *HelpersS) TestAssertFailWithoutExpectedAndMessage(c *check.C) { - checker := &MyChecker{result: false, info: &check.CheckerInfo{Params: []string{"myvalue"}}} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " c\\.Assert\\(1, checker, myComment\\(\"Hello world!\"\\)\\)\n" + - "\\.+ myvalue int = 1\n" + - "\\.+ Hello world!\n\n" - testHelperFailure(c, "Assert(1, checker, msg)", nil, true, log, - func() interface{} { - c.Assert(1, checker, myComment("Hello world!")) - return nil - }) -} - -func (s *HelpersS) TestAssertWithMissingExpected(c *check.C) { - checker := &MyChecker{result: true} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " c\\.Assert\\(1, checker\\)\n" + - "\\.+ Assert\\(myobtained, MyChecker, myexpected\\):\n" + - "\\.+ Wrong number of parameters for MyChecker: " + - "want 3, got 2\n\n" - testHelperFailure(c, "Assert(1, checker, !?)", nil, true, log, - func() interface{} { - c.Assert(1, checker) - return nil - }) -} - -func (s *HelpersS) TestAssertWithError(c *check.C) { - checker := &MyChecker{result: false, error: "Some not so cool data provided!"} - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " c\\.Assert\\(1, checker, 2\\)\n" + - "\\.+ myobtained int = 1\n" + - "\\.+ myexpected int = 2\n" + - "\\.+ Some not so cool data provided!\n\n" - testHelperFailure(c, "Assert(1, checker, 2)", nil, true, log, - func() interface{} { - c.Assert(1, checker, 2) - return nil - }) -} - -func (s *HelpersS) TestAssertWithNilChecker(c *check.C) { - log := "(?s)helpers_test\\.go:[0-9]+:.*\nhelpers_test\\.go:[0-9]+:\n" + - " c\\.Assert\\(1, nil\\)\n" + - "\\.+ Assert\\(obtained, nil!\\?, \\.\\.\\.\\):\n" + - "\\.+ Oops\\.\\. you've provided a nil checker!\n\n" - testHelperFailure(c, "Assert(obtained, nil)", nil, true, log, - func() interface{} { - c.Assert(1, nil) - return nil - }) -} - -// ----------------------------------------------------------------------- -// Ensure that values logged work properly in some interesting cases. - -func (s *HelpersS) TestValueLoggingWithArrays(c *check.C) { - checker := &MyChecker{result: false} - log := "(?s)helpers_test.go:[0-9]+:.*\nhelpers_test.go:[0-9]+:\n" + - " return c\\.Check\\(\\[\\]byte{1, 2}, checker, \\[\\]byte{1, 3}\\)\n" + - "\\.+ myobtained \\[\\]uint8 = \\[\\]byte{0x1, 0x2}\n" + - "\\.+ myexpected \\[\\]uint8 = \\[\\]byte{0x1, 0x3}\n\n" - testHelperFailure(c, "Check([]byte{1}, chk, []byte{3})", false, false, log, - func() interface{} { - return c.Check([]byte{1, 2}, checker, []byte{1, 3}) - }) -} - -func (s *HelpersS) TestValueLoggingWithMultiLine(c *check.C) { - checker := &MyChecker{result: false} - log := "(?s)helpers_test.go:[0-9]+:.*\nhelpers_test.go:[0-9]+:\n" + - " return c\\.Check\\(\"a\\\\nb\\\\n\", checker, \"a\\\\nb\\\\nc\"\\)\n" + - "\\.+ myobtained string = \"\" \\+\n" + - "\\.+ \"a\\\\n\" \\+\n" + - "\\.+ \"b\\\\n\"\n" + - "\\.+ myexpected string = \"\" \\+\n" + - "\\.+ \"a\\\\n\" \\+\n" + - "\\.+ \"b\\\\n\" \\+\n" + - "\\.+ \"c\"\n\n" - testHelperFailure(c, `Check("a\nb\n", chk, "a\nb\nc")`, false, false, log, - func() interface{} { - return c.Check("a\nb\n", checker, "a\nb\nc") - }) -} - -func (s *HelpersS) TestValueLoggingWithMultiLineException(c *check.C) { - // If the newline is at the end of the string, don't log as multi-line. - checker := &MyChecker{result: false} - log := "(?s)helpers_test.go:[0-9]+:.*\nhelpers_test.go:[0-9]+:\n" + - " return c\\.Check\\(\"a b\\\\n\", checker, \"a\\\\nb\"\\)\n" + - "\\.+ myobtained string = \"a b\\\\n\"\n" + - "\\.+ myexpected string = \"\" \\+\n" + - "\\.+ \"a\\\\n\" \\+\n" + - "\\.+ \"b\"\n\n" - testHelperFailure(c, `Check("a b\n", chk, "a\nb")`, false, false, log, - func() interface{} { - return c.Check("a b\n", checker, "a\nb") - }) -} - -// ----------------------------------------------------------------------- -// MakeDir() tests. - -type MkDirHelper struct { - path1 string - path2 string - isDir1 bool - isDir2 bool - isDir3 bool - isDir4 bool -} - -func (s *MkDirHelper) SetUpSuite(c *check.C) { - s.path1 = c.MkDir() - s.isDir1 = isDir(s.path1) -} - -func (s *MkDirHelper) Test(c *check.C) { - s.path2 = c.MkDir() - s.isDir2 = isDir(s.path2) -} - -func (s *MkDirHelper) TearDownSuite(c *check.C) { - s.isDir3 = isDir(s.path1) - s.isDir4 = isDir(s.path2) -} - -func (s *HelpersS) TestMkDir(c *check.C) { - helper := MkDirHelper{} - output := String{} - check.Run(&helper, &check.RunConf{Output: &output}) - c.Assert(output.value, check.Equals, "") - c.Check(helper.isDir1, check.Equals, true) - c.Check(helper.isDir2, check.Equals, true) - c.Check(helper.isDir3, check.Equals, true) - c.Check(helper.isDir4, check.Equals, true) - c.Check(helper.path1, check.Not(check.Equals), - helper.path2) - c.Check(isDir(helper.path1), check.Equals, false) - c.Check(isDir(helper.path2), check.Equals, false) -} - -func isDir(path string) bool { - if stat, err := os.Stat(path); err == nil { - return stat.IsDir() - } - return false -} - -// Concurrent logging should not corrupt the underling buffer. -// Use go test -race to detect the race in this test. -func (s *HelpersS) TestConcurrentLogging(c *check.C) { - defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(runtime.NumCPU())) - var start, stop sync.WaitGroup - start.Add(1) - for i, n := 0, runtime.NumCPU()*2; i < n; i++ { - stop.Add(1) - go func(i int) { - start.Wait() - for j := 0; j < 30; j++ { - c.Logf("Worker %d: line %d", i, j) - } - stop.Done() - }(i) - } - start.Done() - stop.Wait() -} - -// ----------------------------------------------------------------------- -// Test the TestName function - -type TestNameHelper struct { - name1 string - name2 string - name3 string - name4 string - name5 string -} - -func (s *TestNameHelper) SetUpSuite(c *check.C) { s.name1 = c.TestName() } -func (s *TestNameHelper) SetUpTest(c *check.C) { s.name2 = c.TestName() } -func (s *TestNameHelper) Test(c *check.C) { s.name3 = c.TestName() } -func (s *TestNameHelper) TearDownTest(c *check.C) { s.name4 = c.TestName() } -func (s *TestNameHelper) TearDownSuite(c *check.C) { s.name5 = c.TestName() } - -func (s *HelpersS) TestTestName(c *check.C) { - helper := TestNameHelper{} - output := String{} - check.Run(&helper, &check.RunConf{Output: &output}) - c.Check(helper.name1, check.Equals, "") - c.Check(helper.name2, check.Equals, "TestNameHelper.Test") - c.Check(helper.name3, check.Equals, "TestNameHelper.Test") - c.Check(helper.name4, check.Equals, "TestNameHelper.Test") - c.Check(helper.name5, check.Equals, "") -} - -// ----------------------------------------------------------------------- -// A couple of helper functions to test helper functions. :-) - -func testHelperSuccess(c *check.C, name string, expectedResult interface{}, closure func() interface{}) { - var result interface{} - defer (func() { - if err := recover(); err != nil { - panic(err) - } - checkState(c, result, - &expectedState{ - name: name, - result: expectedResult, - failed: false, - log: "", - }) - })() - result = closure() -} - -func testHelperFailure(c *check.C, name string, expectedResult interface{}, shouldStop bool, log string, closure func() interface{}) { - var result interface{} - defer (func() { - if err := recover(); err != nil { - panic(err) - } - checkState(c, result, - &expectedState{ - name: name, - result: expectedResult, - failed: true, - log: log, - }) - })() - result = closure() - if shouldStop { - c.Logf("%s didn't stop when it should", name) - } -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer.go deleted file mode 100644 index e0f7557b..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer.go +++ /dev/null @@ -1,168 +0,0 @@ -package check - -import ( - "bytes" - "go/ast" - "go/parser" - "go/printer" - "go/token" - "os" -) - -func indent(s, with string) (r string) { - eol := true - for i := 0; i != len(s); i++ { - c := s[i] - switch { - case eol && c == '\n' || c == '\r': - case c == '\n' || c == '\r': - eol = true - case eol: - eol = false - s = s[:i] + with + s[i:] - i += len(with) - } - } - return s -} - -func printLine(filename string, line int) (string, error) { - fset := token.NewFileSet() - file, err := os.Open(filename) - if err != nil { - return "", err - } - fnode, err := parser.ParseFile(fset, filename, file, parser.ParseComments) - if err != nil { - return "", err - } - config := &printer.Config{Mode: printer.UseSpaces, Tabwidth: 4} - lp := &linePrinter{fset: fset, fnode: fnode, line: line, config: config} - ast.Walk(lp, fnode) - result := lp.output.Bytes() - // Comments leave \n at the end. - n := len(result) - for n > 0 && result[n-1] == '\n' { - n-- - } - return string(result[:n]), nil -} - -type linePrinter struct { - config *printer.Config - fset *token.FileSet - fnode *ast.File - line int - output bytes.Buffer - stmt ast.Stmt -} - -func (lp *linePrinter) emit() bool { - if lp.stmt != nil { - lp.trim(lp.stmt) - lp.printWithComments(lp.stmt) - lp.stmt = nil - return true - } - return false -} - -func (lp *linePrinter) printWithComments(n ast.Node) { - nfirst := lp.fset.Position(n.Pos()).Line - nlast := lp.fset.Position(n.End()).Line - for _, g := range lp.fnode.Comments { - cfirst := lp.fset.Position(g.Pos()).Line - clast := lp.fset.Position(g.End()).Line - if clast == nfirst-1 && lp.fset.Position(n.Pos()).Column == lp.fset.Position(g.Pos()).Column { - for _, c := range g.List { - lp.output.WriteString(c.Text) - lp.output.WriteByte('\n') - } - } - if cfirst >= nfirst && cfirst <= nlast && n.End() <= g.List[0].Slash { - // The printer will not include the comment if it starts past - // the node itself. Trick it into printing by overlapping the - // slash with the end of the statement. - g.List[0].Slash = n.End() - 1 - } - } - node := &printer.CommentedNode{n, lp.fnode.Comments} - lp.config.Fprint(&lp.output, lp.fset, node) -} - -func (lp *linePrinter) Visit(n ast.Node) (w ast.Visitor) { - if n == nil { - if lp.output.Len() == 0 { - lp.emit() - } - return nil - } - first := lp.fset.Position(n.Pos()).Line - last := lp.fset.Position(n.End()).Line - if first <= lp.line && last >= lp.line { - // Print the innermost statement containing the line. - if stmt, ok := n.(ast.Stmt); ok { - if _, ok := n.(*ast.BlockStmt); !ok { - lp.stmt = stmt - } - } - if first == lp.line && lp.emit() { - return nil - } - return lp - } - return nil -} - -func (lp *linePrinter) trim(n ast.Node) bool { - stmt, ok := n.(ast.Stmt) - if !ok { - return true - } - line := lp.fset.Position(n.Pos()).Line - if line != lp.line { - return false - } - switch stmt := stmt.(type) { - case *ast.IfStmt: - stmt.Body = lp.trimBlock(stmt.Body) - case *ast.SwitchStmt: - stmt.Body = lp.trimBlock(stmt.Body) - case *ast.TypeSwitchStmt: - stmt.Body = lp.trimBlock(stmt.Body) - case *ast.CaseClause: - stmt.Body = lp.trimList(stmt.Body) - case *ast.CommClause: - stmt.Body = lp.trimList(stmt.Body) - case *ast.BlockStmt: - stmt.List = lp.trimList(stmt.List) - } - return true -} - -func (lp *linePrinter) trimBlock(stmt *ast.BlockStmt) *ast.BlockStmt { - if !lp.trim(stmt) { - return lp.emptyBlock(stmt) - } - stmt.Rbrace = stmt.Lbrace - return stmt -} - -func (lp *linePrinter) trimList(stmts []ast.Stmt) []ast.Stmt { - for i := 0; i != len(stmts); i++ { - if !lp.trim(stmts[i]) { - stmts[i] = lp.emptyStmt(stmts[i]) - break - } - } - return stmts -} - -func (lp *linePrinter) emptyStmt(n ast.Node) *ast.ExprStmt { - return &ast.ExprStmt{&ast.Ellipsis{n.Pos(), nil}} -} - -func (lp *linePrinter) emptyBlock(n ast.Node) *ast.BlockStmt { - p := n.Pos() - return &ast.BlockStmt{p, []ast.Stmt{lp.emptyStmt(n)}, p} -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer_test.go deleted file mode 100644 index 47f94a7f..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/printer_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package check_test - -import ( - . "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" -) - -var _ = Suite(&PrinterS{}) - -type PrinterS struct{} - -func (s *PrinterS) TestCountSuite(c *C) { - suitesRun += 1 -} - -var printTestFuncLine int - -func init() { - printTestFuncLine = getMyLine() + 3 -} - -func printTestFunc() { - println(1) // Comment1 - if 2 == 2 { // Comment2 - println(3) // Comment3 - } - switch 5 { - case 6: - println(6) // Comment6 - println(7) - } - switch interface{}(9).(type) { // Comment9 - case int: - println(10) - println(11) - } - select { - case <-(chan bool)(nil): - println(14) - println(15) - default: - println(16) - println(17) - } - println(19, - 20) - _ = func() { - println(21) - println(22) - } - println(24, func() { - println(25) - }) - // Leading comment - // with multiple lines. - println(29) // Comment29 -} - -var printLineTests = []struct { - line int - output string -}{ - {1, "println(1) // Comment1"}, - {2, "if 2 == 2 { // Comment2\n ...\n}"}, - {3, "println(3) // Comment3"}, - {5, "switch 5 {\n...\n}"}, - {6, "case 6:\n println(6) // Comment6\n ..."}, - {7, "println(7)"}, - {9, "switch interface{}(9).(type) { // Comment9\n...\n}"}, - {10, "case int:\n println(10)\n ..."}, - {14, "case <-(chan bool)(nil):\n println(14)\n ..."}, - {15, "println(15)"}, - {16, "default:\n println(16)\n ..."}, - {17, "println(17)"}, - {19, "println(19,\n 20)"}, - {20, "println(19,\n 20)"}, - {21, "_ = func() {\n println(21)\n println(22)\n}"}, - {22, "println(22)"}, - {24, "println(24, func() {\n println(25)\n})"}, - {25, "println(25)"}, - {26, "println(24, func() {\n println(25)\n})"}, - {29, "// Leading comment\n// with multiple lines.\nprintln(29) // Comment29"}, -} - -func (s *PrinterS) TestPrintLine(c *C) { - for _, test := range printLineTests { - output, err := PrintLine("printer_test.go", printTestFuncLine+test.line) - c.Assert(err, IsNil) - c.Assert(output, Equals, test.output) - } -} - -var indentTests = []struct { - in, out string -}{ - {"", ""}, - {"\n", "\n"}, - {"a", ">>>a"}, - {"a\n", ">>>a\n"}, - {"a\nb", ">>>a\n>>>b"}, - {" ", ">>> "}, -} - -func (s *PrinterS) TestIndent(c *C) { - for _, test := range indentTests { - out := Indent(test.in, ">>>") - c.Assert(out, Equals, test.out) - } - -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run.go deleted file mode 100644 index da8fd798..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run.go +++ /dev/null @@ -1,175 +0,0 @@ -package check - -import ( - "bufio" - "flag" - "fmt" - "os" - "testing" - "time" -) - -// ----------------------------------------------------------------------- -// Test suite registry. - -var allSuites []interface{} - -// Suite registers the given value as a test suite to be run. Any methods -// starting with the Test prefix in the given value will be considered as -// a test method. -func Suite(suite interface{}) interface{} { - allSuites = append(allSuites, suite) - return suite -} - -// ----------------------------------------------------------------------- -// Public running interface. - -var ( - oldFilterFlag = flag.String("gocheck.f", "", "Regular expression selecting which tests and/or suites to run") - oldVerboseFlag = flag.Bool("gocheck.v", false, "Verbose mode") - oldStreamFlag = flag.Bool("gocheck.vv", false, "Super verbose mode (disables output caching)") - oldBenchFlag = flag.Bool("gocheck.b", false, "Run benchmarks") - oldBenchTime = flag.Duration("gocheck.btime", 1*time.Second, "approximate run time for each benchmark") - oldListFlag = flag.Bool("gocheck.list", false, "List the names of all tests that will be run") - oldWorkFlag = flag.Bool("gocheck.work", false, "Display and do not remove the test working directory") - - newFilterFlag = flag.String("check.f", "", "Regular expression selecting which tests and/or suites to run") - newVerboseFlag = flag.Bool("check.v", false, "Verbose mode") - newStreamFlag = flag.Bool("check.vv", false, "Super verbose mode (disables output caching)") - newBenchFlag = flag.Bool("check.b", false, "Run benchmarks") - newBenchTime = flag.Duration("check.btime", 1*time.Second, "approximate run time for each benchmark") - newBenchMem = flag.Bool("check.bmem", false, "Report memory benchmarks") - newListFlag = flag.Bool("check.list", false, "List the names of all tests that will be run") - newWorkFlag = flag.Bool("check.work", false, "Display and do not remove the test working directory") -) - -// TestingT runs all test suites registered with the Suite function, -// printing results to stdout, and reporting any failures back to -// the "testing" package. -func TestingT(testingT *testing.T) { - benchTime := *newBenchTime - if benchTime == 1*time.Second { - benchTime = *oldBenchTime - } - conf := &RunConf{ - Filter: *oldFilterFlag + *newFilterFlag, - Verbose: *oldVerboseFlag || *newVerboseFlag, - Stream: *oldStreamFlag || *newStreamFlag, - Benchmark: *oldBenchFlag || *newBenchFlag, - BenchmarkTime: benchTime, - BenchmarkMem: *newBenchMem, - KeepWorkDir: *oldWorkFlag || *newWorkFlag, - } - if *oldListFlag || *newListFlag { - w := bufio.NewWriter(os.Stdout) - for _, name := range ListAll(conf) { - fmt.Fprintln(w, name) - } - w.Flush() - return - } - result := RunAll(conf) - println(result.String()) - if !result.Passed() { - testingT.Fail() - } -} - -// RunAll runs all test suites registered with the Suite function, using the -// provided run configuration. -func RunAll(runConf *RunConf) *Result { - result := Result{} - for _, suite := range allSuites { - result.Add(Run(suite, runConf)) - } - return &result -} - -// Run runs the provided test suite using the provided run configuration. -func Run(suite interface{}, runConf *RunConf) *Result { - runner := newSuiteRunner(suite, runConf) - return runner.run() -} - -// ListAll returns the names of all the test functions registered with the -// Suite function that will be run with the provided run configuration. -func ListAll(runConf *RunConf) []string { - var names []string - for _, suite := range allSuites { - names = append(names, List(suite, runConf)...) - } - return names -} - -// List returns the names of the test functions in the given -// suite that will be run with the provided run configuration. -func List(suite interface{}, runConf *RunConf) []string { - var names []string - runner := newSuiteRunner(suite, runConf) - for _, t := range runner.tests { - names = append(names, t.String()) - } - return names -} - -// ----------------------------------------------------------------------- -// Result methods. - -func (r *Result) Add(other *Result) { - r.Succeeded += other.Succeeded - r.Skipped += other.Skipped - r.Failed += other.Failed - r.Panicked += other.Panicked - r.FixturePanicked += other.FixturePanicked - r.ExpectedFailures += other.ExpectedFailures - r.Missed += other.Missed - if r.WorkDir != "" && other.WorkDir != "" { - r.WorkDir += ":" + other.WorkDir - } else if other.WorkDir != "" { - r.WorkDir = other.WorkDir - } -} - -func (r *Result) Passed() bool { - return (r.Failed == 0 && r.Panicked == 0 && - r.FixturePanicked == 0 && r.Missed == 0 && - r.RunError == nil) -} - -func (r *Result) String() string { - if r.RunError != nil { - return "ERROR: " + r.RunError.Error() - } - - var value string - if r.Failed == 0 && r.Panicked == 0 && r.FixturePanicked == 0 && - r.Missed == 0 { - value = "OK: " - } else { - value = "OOPS: " - } - value += fmt.Sprintf("%d passed", r.Succeeded) - if r.Skipped != 0 { - value += fmt.Sprintf(", %d skipped", r.Skipped) - } - if r.ExpectedFailures != 0 { - value += fmt.Sprintf(", %d expected failures", r.ExpectedFailures) - } - if r.Failed != 0 { - value += fmt.Sprintf(", %d FAILED", r.Failed) - } - if r.Panicked != 0 { - value += fmt.Sprintf(", %d PANICKED", r.Panicked) - } - if r.FixturePanicked != 0 { - value += fmt.Sprintf(", %d FIXTURE-PANICKED", r.FixturePanicked) - } - if r.Missed != 0 { - value += fmt.Sprintf(", %d MISSED", r.Missed) - } - if r.WorkDir != "" { - value += "\nWORK=" + r.WorkDir - } - return value -} diff --git a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run_test.go b/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run_test.go deleted file mode 100644 index 4444cacf..00000000 --- a/vendor/github.com/magiconair/properties/_third_party/gopkg.in/check.v1/run_test.go +++ /dev/null @@ -1,419 +0,0 @@ -// These tests verify the test running logic. - -package check_test - -import ( - "errors" - . "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" - "os" - "sync" -) - -var runnerS = Suite(&RunS{}) - -type RunS struct{} - -func (s *RunS) TestCountSuite(c *C) { - suitesRun += 1 -} - -// ----------------------------------------------------------------------- -// Tests ensuring result counting works properly. - -func (s *RunS) TestSuccess(c *C) { - output := String{} - result := Run(&SuccessHelper{}, &RunConf{Output: &output}) - c.Check(result.Succeeded, Equals, 1) - c.Check(result.Failed, Equals, 0) - c.Check(result.Skipped, Equals, 0) - c.Check(result.Panicked, Equals, 0) - c.Check(result.FixturePanicked, Equals, 0) - c.Check(result.Missed, Equals, 0) - c.Check(result.RunError, IsNil) -} - -func (s *RunS) TestFailure(c *C) { - output := String{} - result := Run(&FailHelper{}, &RunConf{Output: &output}) - c.Check(result.Succeeded, Equals, 0) - c.Check(result.Failed, Equals, 1) - c.Check(result.Skipped, Equals, 0) - c.Check(result.Panicked, Equals, 0) - c.Check(result.FixturePanicked, Equals, 0) - c.Check(result.Missed, Equals, 0) - c.Check(result.RunError, IsNil) -} - -func (s *RunS) TestFixture(c *C) { - output := String{} - result := Run(&FixtureHelper{}, &RunConf{Output: &output}) - c.Check(result.Succeeded, Equals, 2) - c.Check(result.Failed, Equals, 0) - c.Check(result.Skipped, Equals, 0) - c.Check(result.Panicked, Equals, 0) - c.Check(result.FixturePanicked, Equals, 0) - c.Check(result.Missed, Equals, 0) - c.Check(result.RunError, IsNil) -} - -func (s *RunS) TestPanicOnTest(c *C) { - output := String{} - helper := &FixtureHelper{panicOn: "Test1"} - result := Run(helper, &RunConf{Output: &output}) - c.Check(result.Succeeded, Equals, 1) - c.Check(result.Failed, Equals, 0) - c.Check(result.Skipped, Equals, 0) - c.Check(result.Panicked, Equals, 1) - c.Check(result.FixturePanicked, Equals, 0) - c.Check(result.Missed, Equals, 0) - c.Check(result.RunError, IsNil) -} - -func (s *RunS) TestPanicOnSetUpTest(c *C) { - output := String{} - helper := &FixtureHelper{panicOn: "SetUpTest"} - result := Run(helper, &RunConf{Output: &output}) - c.Check(result.Succeeded, Equals, 0) - c.Check(result.Failed, Equals, 0) - c.Check(result.Skipped, Equals, 0) - c.Check(result.Panicked, Equals, 0) - c.Check(result.FixturePanicked, Equals, 1) - c.Check(result.Missed, Equals, 2) - c.Check(result.RunError, IsNil) -} - -func (s *RunS) TestPanicOnSetUpSuite(c *C) { - output := String{} - helper := &FixtureHelper{panicOn: "SetUpSuite"} - result := Run(helper, &RunConf{Output: &output}) - c.Check(result.Succeeded, Equals, 0) - c.Check(result.Failed, Equals, 0) - c.Check(result.Skipped, Equals, 0) - c.Check(result.Panicked, Equals, 0) - c.Check(result.FixturePanicked, Equals, 1) - c.Check(result.Missed, Equals, 2) - c.Check(result.RunError, IsNil) -} - -// ----------------------------------------------------------------------- -// Check result aggregation. - -func (s *RunS) TestAdd(c *C) { - result := &Result{ - Succeeded: 1, - Skipped: 2, - Failed: 3, - Panicked: 4, - FixturePanicked: 5, - Missed: 6, - ExpectedFailures: 7, - } - result.Add(&Result{ - Succeeded: 10, - Skipped: 20, - Failed: 30, - Panicked: 40, - FixturePanicked: 50, - Missed: 60, - ExpectedFailures: 70, - }) - c.Check(result.Succeeded, Equals, 11) - c.Check(result.Skipped, Equals, 22) - c.Check(result.Failed, Equals, 33) - c.Check(result.Panicked, Equals, 44) - c.Check(result.FixturePanicked, Equals, 55) - c.Check(result.Missed, Equals, 66) - c.Check(result.ExpectedFailures, Equals, 77) - c.Check(result.RunError, IsNil) -} - -// ----------------------------------------------------------------------- -// Check the Passed() method. - -func (s *RunS) TestPassed(c *C) { - c.Assert((&Result{}).Passed(), Equals, true) - c.Assert((&Result{Succeeded: 1}).Passed(), Equals, true) - c.Assert((&Result{Skipped: 1}).Passed(), Equals, true) - c.Assert((&Result{Failed: 1}).Passed(), Equals, false) - c.Assert((&Result{Panicked: 1}).Passed(), Equals, false) - c.Assert((&Result{FixturePanicked: 1}).Passed(), Equals, false) - c.Assert((&Result{Missed: 1}).Passed(), Equals, false) - c.Assert((&Result{RunError: errors.New("!")}).Passed(), Equals, false) -} - -// ----------------------------------------------------------------------- -// Check that result printing is working correctly. - -func (s *RunS) TestPrintSuccess(c *C) { - result := &Result{Succeeded: 5} - c.Check(result.String(), Equals, "OK: 5 passed") -} - -func (s *RunS) TestPrintFailure(c *C) { - result := &Result{Failed: 5} - c.Check(result.String(), Equals, "OOPS: 0 passed, 5 FAILED") -} - -func (s *RunS) TestPrintSkipped(c *C) { - result := &Result{Skipped: 5} - c.Check(result.String(), Equals, "OK: 0 passed, 5 skipped") -} - -func (s *RunS) TestPrintExpectedFailures(c *C) { - result := &Result{ExpectedFailures: 5} - c.Check(result.String(), Equals, "OK: 0 passed, 5 expected failures") -} - -func (s *RunS) TestPrintPanicked(c *C) { - result := &Result{Panicked: 5} - c.Check(result.String(), Equals, "OOPS: 0 passed, 5 PANICKED") -} - -func (s *RunS) TestPrintFixturePanicked(c *C) { - result := &Result{FixturePanicked: 5} - c.Check(result.String(), Equals, "OOPS: 0 passed, 5 FIXTURE-PANICKED") -} - -func (s *RunS) TestPrintMissed(c *C) { - result := &Result{Missed: 5} - c.Check(result.String(), Equals, "OOPS: 0 passed, 5 MISSED") -} - -func (s *RunS) TestPrintAll(c *C) { - result := &Result{Succeeded: 1, Skipped: 2, ExpectedFailures: 3, - Panicked: 4, FixturePanicked: 5, Missed: 6} - c.Check(result.String(), Equals, - "OOPS: 1 passed, 2 skipped, 3 expected failures, 4 PANICKED, "+ - "5 FIXTURE-PANICKED, 6 MISSED") -} - -func (s *RunS) TestPrintRunError(c *C) { - result := &Result{Succeeded: 1, Failed: 1, - RunError: errors.New("Kaboom!")} - c.Check(result.String(), Equals, "ERROR: Kaboom!") -} - -// ----------------------------------------------------------------------- -// Verify that the method pattern flag works correctly. - -func (s *RunS) TestFilterTestName(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Filter: "Test[91]"} - Run(&helper, &runConf) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 5) -} - -func (s *RunS) TestFilterTestNameWithAll(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Filter: ".*"} - Run(&helper, &runConf) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "SetUpTest") - c.Check(helper.calls[5], Equals, "Test2") - c.Check(helper.calls[6], Equals, "TearDownTest") - c.Check(helper.calls[7], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 8) -} - -func (s *RunS) TestFilterSuiteName(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Filter: "FixtureHelper"} - Run(&helper, &runConf) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test1") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "SetUpTest") - c.Check(helper.calls[5], Equals, "Test2") - c.Check(helper.calls[6], Equals, "TearDownTest") - c.Check(helper.calls[7], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 8) -} - -func (s *RunS) TestFilterSuiteNameAndTestName(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Filter: "FixtureHelper\\.Test2"} - Run(&helper, &runConf) - c.Check(helper.calls[0], Equals, "SetUpSuite") - c.Check(helper.calls[1], Equals, "SetUpTest") - c.Check(helper.calls[2], Equals, "Test2") - c.Check(helper.calls[3], Equals, "TearDownTest") - c.Check(helper.calls[4], Equals, "TearDownSuite") - c.Check(len(helper.calls), Equals, 5) -} - -func (s *RunS) TestFilterAllOut(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Filter: "NotFound"} - Run(&helper, &runConf) - c.Check(len(helper.calls), Equals, 0) -} - -func (s *RunS) TestRequirePartialMatch(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Filter: "est"} - Run(&helper, &runConf) - c.Check(len(helper.calls), Equals, 8) -} - -func (s *RunS) TestFilterError(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Filter: "]["} - result := Run(&helper, &runConf) - c.Check(result.String(), Equals, - "ERROR: Bad filter expression: error parsing regexp: missing closing ]: `[`") - c.Check(len(helper.calls), Equals, 0) -} - -// ----------------------------------------------------------------------- -// Verify that List works correctly. - -func (s *RunS) TestListFiltered(c *C) { - names := List(&FixtureHelper{}, &RunConf{Filter: "1"}) - c.Assert(names, DeepEquals, []string{ - "FixtureHelper.Test1", - }) -} - -func (s *RunS) TestList(c *C) { - names := List(&FixtureHelper{}, &RunConf{}) - c.Assert(names, DeepEquals, []string{ - "FixtureHelper.Test1", - "FixtureHelper.Test2", - }) -} - -// ----------------------------------------------------------------------- -// Verify that verbose mode prints tests which pass as well. - -func (s *RunS) TestVerboseMode(c *C) { - helper := FixtureHelper{} - output := String{} - runConf := RunConf{Output: &output, Verbose: true} - Run(&helper, &runConf) - - expected := "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Test1\t *[.0-9]+s\n" + - "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Test2\t *[.0-9]+s\n" - - c.Assert(output.value, Matches, expected) -} - -func (s *RunS) TestVerboseModeWithFailBeforePass(c *C) { - helper := FixtureHelper{panicOn: "Test1"} - output := String{} - runConf := RunConf{Output: &output, Verbose: true} - Run(&helper, &runConf) - - expected := "(?s).*PANIC.*\n-+\n" + // Should have an extra line. - "PASS: check_test\\.go:[0-9]+: FixtureHelper\\.Test2\t *[.0-9]+s\n" - - c.Assert(output.value, Matches, expected) -} - -// ----------------------------------------------------------------------- -// Verify the stream output mode. In this mode there's no output caching. - -type StreamHelper struct { - l2 sync.Mutex - l3 sync.Mutex -} - -func (s *StreamHelper) SetUpSuite(c *C) { - c.Log("0") -} - -func (s *StreamHelper) Test1(c *C) { - c.Log("1") - s.l2.Lock() - s.l3.Lock() - go func() { - s.l2.Lock() // Wait for "2". - c.Log("3") - s.l3.Unlock() - }() -} - -func (s *StreamHelper) Test2(c *C) { - c.Log("2") - s.l2.Unlock() - s.l3.Lock() // Wait for "3". - c.Fail() - c.Log("4") -} - -func (s *RunS) TestStreamMode(c *C) { - helper := &StreamHelper{} - output := String{} - runConf := RunConf{Output: &output, Stream: true} - Run(helper, &runConf) - - expected := "START: run_test\\.go:[0-9]+: StreamHelper\\.SetUpSuite\n0\n" + - "PASS: run_test\\.go:[0-9]+: StreamHelper\\.SetUpSuite\t *[.0-9]+s\n\n" + - "START: run_test\\.go:[0-9]+: StreamHelper\\.Test1\n1\n" + - "PASS: run_test\\.go:[0-9]+: StreamHelper\\.Test1\t *[.0-9]+s\n\n" + - "START: run_test\\.go:[0-9]+: StreamHelper\\.Test2\n2\n3\n4\n" + - "FAIL: run_test\\.go:[0-9]+: StreamHelper\\.Test2\n\n" - - c.Assert(output.value, Matches, expected) -} - -type StreamMissHelper struct{} - -func (s *StreamMissHelper) SetUpSuite(c *C) { - c.Log("0") - c.Fail() -} - -func (s *StreamMissHelper) Test1(c *C) { - c.Log("1") -} - -func (s *RunS) TestStreamModeWithMiss(c *C) { - helper := &StreamMissHelper{} - output := String{} - runConf := RunConf{Output: &output, Stream: true} - Run(helper, &runConf) - - expected := "START: run_test\\.go:[0-9]+: StreamMissHelper\\.SetUpSuite\n0\n" + - "FAIL: run_test\\.go:[0-9]+: StreamMissHelper\\.SetUpSuite\n\n" + - "START: run_test\\.go:[0-9]+: StreamMissHelper\\.Test1\n" + - "MISS: run_test\\.go:[0-9]+: StreamMissHelper\\.Test1\n\n" - - c.Assert(output.value, Matches, expected) -} - -// ----------------------------------------------------------------------- -// Verify that that the keep work dir request indeed does so. - -type WorkDirSuite struct{} - -func (s *WorkDirSuite) Test(c *C) { - c.MkDir() -} - -func (s *RunS) TestKeepWorkDir(c *C) { - output := String{} - runConf := RunConf{Output: &output, Verbose: true, KeepWorkDir: true} - result := Run(&WorkDirSuite{}, &runConf) - - c.Assert(result.String(), Matches, ".*\nWORK="+result.WorkDir) - - stat, err := os.Stat(result.WorkDir) - c.Assert(err, IsNil) - c.Assert(stat.IsDir(), Equals, true) -} diff --git a/vendor/github.com/magiconair/properties/benchmark_test.go b/vendor/github.com/magiconair/properties/benchmark_test.go deleted file mode 100644 index b2019e10..00000000 --- a/vendor/github.com/magiconair/properties/benchmark_test.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package properties - -import ( - "fmt" - "testing" -) - -// Benchmarks the decoder by creating a property file with 1000 key/value pairs. -func BenchmarkLoad(b *testing.B) { - input := "" - for i := 0; i < 1000; i++ { - input += fmt.Sprintf("key%d=value%d\n", i, i) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - Load([]byte(input), ISO_8859_1) - } -} diff --git a/vendor/github.com/magiconair/properties/decode.go b/vendor/github.com/magiconair/properties/decode.go index cef8c3a3..0a961bb0 100644 --- a/vendor/github.com/magiconair/properties/decode.go +++ b/vendor/github.com/magiconair/properties/decode.go @@ -1,3 +1,7 @@ +// Copyright 2017 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package properties import ( @@ -154,16 +158,16 @@ func dec(p *Properties, key string, def *string, opts map[string]string, v refle // keydef returns the property key and the default value based on the // name of the struct field and the options in the tag. keydef := func(f reflect.StructField) (string, *string, map[string]string) { - key, opts := parseTag(f.Tag.Get("properties")) + _key, _opts := parseTag(f.Tag.Get("properties")) - var def *string - if d, ok := opts["default"]; ok { - def = &d + var _def *string + if d, ok := _opts["default"]; ok { + _def = &d } - if key != "" { - return key, def, opts + if _key != "" { + return _key, _def, _opts } - return f.Name, def, opts + return f.Name, _def, _opts } switch { @@ -186,7 +190,7 @@ func dec(p *Properties, key string, def *string, opts map[string]string, v refle fv := v.Field(i) fk, def, opts := keydef(t.Field(i)) if !fv.CanSet() { - return fmt.Errorf("cannot set ", t.Field(i).Name) + return fmt.Errorf("cannot set %s", t.Field(i).Name) } if fk == "-" { continue @@ -219,7 +223,7 @@ func dec(p *Properties, key string, def *string, opts map[string]string, v refle case isMap(t): valT := t.Elem() m := reflect.MakeMap(t) - for postfix, _ := range p.FilterStripPrefix(key + ".").m { + for postfix := range p.FilterStripPrefix(key + ".").m { pp := strings.SplitN(postfix, ".", 2) mk, mv := pp[0], reflect.New(valT) if err := dec(p, key+"."+mk, nil, nil, mv); err != nil { @@ -270,7 +274,6 @@ func isArray(t reflect.Type) bool { return t.Kind() == reflect.Array || t.Kin func isBool(t reflect.Type) bool { return t.Kind() == reflect.Bool } func isDuration(t reflect.Type) bool { return t == reflect.TypeOf(time.Second) } func isMap(t reflect.Type) bool { return t.Kind() == reflect.Map } -func isNumeric(t reflect.Type) bool { return isInt(t) || isUint(t) || isFloat(t) } func isPtr(t reflect.Type) bool { return t.Kind() == reflect.Ptr } func isString(t reflect.Type) bool { return t.Kind() == reflect.String } func isStruct(t reflect.Type) bool { return t.Kind() == reflect.Struct } diff --git a/vendor/github.com/magiconair/properties/decode_test.go b/vendor/github.com/magiconair/properties/decode_test.go deleted file mode 100644 index 4115fbbb..00000000 --- a/vendor/github.com/magiconair/properties/decode_test.go +++ /dev/null @@ -1,295 +0,0 @@ -package properties - -import ( - "reflect" - "testing" - "time" -) - -func TestDecodeValues(t *testing.T) { - type S struct { - S string - BT bool - BF bool - I int - I8 int8 - I16 int16 - I32 int32 - I64 int64 - U uint - U8 uint8 - U16 uint16 - U32 uint32 - U64 uint64 - F32 float32 - F64 float64 - D time.Duration - TM time.Time - } - in := ` - S=abc - BT=true - BF=false - I=-1 - I8=-8 - I16=-16 - I32=-32 - I64=-64 - U=1 - U8=8 - U16=16 - U32=32 - U64=64 - F32=3.2 - F64=6.4 - D=5s - TM=2015-01-02T12:34:56Z - ` - out := &S{ - S: "abc", - BT: true, - BF: false, - I: -1, - I8: -8, - I16: -16, - I32: -32, - I64: -64, - U: 1, - U8: 8, - U16: 16, - U32: 32, - U64: 64, - F32: 3.2, - F64: 6.4, - D: 5 * time.Second, - TM: tm(t, time.RFC3339, "2015-01-02T12:34:56Z"), - } - testDecode(t, in, &S{}, out) -} - -func TestDecodeValueDefaults(t *testing.T) { - type S struct { - S string `properties:",default=abc"` - BT bool `properties:",default=true"` - BF bool `properties:",default=false"` - I int `properties:",default=-1"` - I8 int8 `properties:",default=-8"` - I16 int16 `properties:",default=-16"` - I32 int32 `properties:",default=-32"` - I64 int64 `properties:",default=-64"` - U uint `properties:",default=1"` - U8 uint8 `properties:",default=8"` - U16 uint16 `properties:",default=16"` - U32 uint32 `properties:",default=32"` - U64 uint64 `properties:",default=64"` - F32 float32 `properties:",default=3.2"` - F64 float64 `properties:",default=6.4"` - D time.Duration `properties:",default=5s"` - TM time.Time `properties:",default=2015-01-02T12:34:56Z"` - } - out := &S{ - S: "abc", - BT: true, - BF: false, - I: -1, - I8: -8, - I16: -16, - I32: -32, - I64: -64, - U: 1, - U8: 8, - U16: 16, - U32: 32, - U64: 64, - F32: 3.2, - F64: 6.4, - D: 5 * time.Second, - TM: tm(t, time.RFC3339, "2015-01-02T12:34:56Z"), - } - testDecode(t, "", &S{}, out) -} - -func TestDecodeArrays(t *testing.T) { - type S struct { - S []string - B []bool - I []int - I8 []int8 - I16 []int16 - I32 []int32 - I64 []int64 - U []uint - U8 []uint8 - U16 []uint16 - U32 []uint32 - U64 []uint64 - F32 []float32 - F64 []float64 - D []time.Duration - TM []time.Time - } - in := ` - S=a;b - B=true;false - I=-1;-2 - I8=-8;-9 - I16=-16;-17 - I32=-32;-33 - I64=-64;-65 - U=1;2 - U8=8;9 - U16=16;17 - U32=32;33 - U64=64;65 - F32=3.2;3.3 - F64=6.4;6.5 - D=4s;5s - TM=2015-01-01T00:00:00Z;2016-01-01T00:00:00Z - ` - out := &S{ - S: []string{"a", "b"}, - B: []bool{true, false}, - I: []int{-1, -2}, - I8: []int8{-8, -9}, - I16: []int16{-16, -17}, - I32: []int32{-32, -33}, - I64: []int64{-64, -65}, - U: []uint{1, 2}, - U8: []uint8{8, 9}, - U16: []uint16{16, 17}, - U32: []uint32{32, 33}, - U64: []uint64{64, 65}, - F32: []float32{3.2, 3.3}, - F64: []float64{6.4, 6.5}, - D: []time.Duration{4 * time.Second, 5 * time.Second}, - TM: []time.Time{tm(t, time.RFC3339, "2015-01-01T00:00:00Z"), tm(t, time.RFC3339, "2016-01-01T00:00:00Z")}, - } - testDecode(t, in, &S{}, out) -} - -func TestDecodeArrayDefaults(t *testing.T) { - type S struct { - S []string `properties:",default=a;b"` - B []bool `properties:",default=true;false"` - I []int `properties:",default=-1;-2"` - I8 []int8 `properties:",default=-8;-9"` - I16 []int16 `properties:",default=-16;-17"` - I32 []int32 `properties:",default=-32;-33"` - I64 []int64 `properties:",default=-64;-65"` - U []uint `properties:",default=1;2"` - U8 []uint8 `properties:",default=8;9"` - U16 []uint16 `properties:",default=16;17"` - U32 []uint32 `properties:",default=32;33"` - U64 []uint64 `properties:",default=64;65"` - F32 []float32 `properties:",default=3.2;3.3"` - F64 []float64 `properties:",default=6.4;6.5"` - D []time.Duration `properties:",default=4s;5s"` - TM []time.Time `properties:",default=2015-01-01T00:00:00Z;2016-01-01T00:00:00Z"` - } - out := &S{ - S: []string{"a", "b"}, - B: []bool{true, false}, - I: []int{-1, -2}, - I8: []int8{-8, -9}, - I16: []int16{-16, -17}, - I32: []int32{-32, -33}, - I64: []int64{-64, -65}, - U: []uint{1, 2}, - U8: []uint8{8, 9}, - U16: []uint16{16, 17}, - U32: []uint32{32, 33}, - U64: []uint64{64, 65}, - F32: []float32{3.2, 3.3}, - F64: []float64{6.4, 6.5}, - D: []time.Duration{4 * time.Second, 5 * time.Second}, - TM: []time.Time{tm(t, time.RFC3339, "2015-01-01T00:00:00Z"), tm(t, time.RFC3339, "2016-01-01T00:00:00Z")}, - } - testDecode(t, "", &S{}, out) -} - -func TestDecodeSkipUndef(t *testing.T) { - type S struct { - X string `properties:"-"` - Undef string `properties:",default=some value"` - } - in := `X=ignore` - out := &S{"", "some value"} - testDecode(t, in, &S{}, out) -} - -func TestDecodeStruct(t *testing.T) { - type A struct { - S string - T string `properties:"t"` - U string `properties:"u,default=uuu"` - } - type S struct { - A A - B A `properties:"b"` - } - in := ` - A.S=sss - A.t=ttt - b.S=SSS - b.t=TTT - ` - out := &S{ - A{S: "sss", T: "ttt", U: "uuu"}, - A{S: "SSS", T: "TTT", U: "uuu"}, - } - testDecode(t, in, &S{}, out) -} - -func TestDecodeMap(t *testing.T) { - type S struct { - A string `properties:"a"` - } - type X struct { - A map[string]string - B map[string][]string - C map[string]map[string]string - D map[string]S - E map[string]int - F map[string]int `properties:"-"` - } - in := ` - A.foo=bar - A.bar=bang - B.foo=a;b;c - B.bar=1;2;3 - C.foo.one=1 - C.foo.two=2 - C.bar.three=3 - C.bar.four=4 - D.foo.a=bar - ` - out := &X{ - A: map[string]string{"foo": "bar", "bar": "bang"}, - B: map[string][]string{"foo": []string{"a", "b", "c"}, "bar": []string{"1", "2", "3"}}, - C: map[string]map[string]string{"foo": map[string]string{"one": "1", "two": "2"}, "bar": map[string]string{"three": "3", "four": "4"}}, - D: map[string]S{"foo": S{"bar"}}, - E: map[string]int{}, - } - testDecode(t, in, &X{}, out) -} - -func testDecode(t *testing.T, in string, v, out interface{}) { - p, err := parse(in) - if err != nil { - t.Fatalf("got %v want nil", err) - } - if err := p.Decode(v); err != nil { - t.Fatalf("got %v want nil", err) - } - if got, want := v, out; !reflect.DeepEqual(got, want) { - t.Fatalf("\ngot %+v\nwant %+v", got, want) - } -} - -func tm(t *testing.T, layout, s string) time.Time { - tm, err := time.Parse(layout, s) - if err != nil { - t.Fatalf("got %v want nil", err) - } - return tm -} diff --git a/vendor/github.com/magiconair/properties/doc.go b/vendor/github.com/magiconair/properties/doc.go index 1d3fb866..36c83680 100644 --- a/vendor/github.com/magiconair/properties/doc.go +++ b/vendor/github.com/magiconair/properties/doc.go @@ -1,4 +1,4 @@ -// Copyright 2015 Frank Schroeder. All rights reserved. +// Copyright 2017 Frank Schroeder. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/vendor/github.com/magiconair/properties/example_test.go b/vendor/github.com/magiconair/properties/example_test.go deleted file mode 100644 index 38bf04f5..00000000 --- a/vendor/github.com/magiconair/properties/example_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package properties - -import ( - "fmt" - "log" -) - -func ExampleLoad_iso88591() { - buf := []byte("key = ISO-8859-1 value with unicode literal \\u2318 and umlaut \xE4") // 0xE4 == ä - p, _ := Load(buf, ISO_8859_1) - v, ok := p.Get("key") - fmt.Println(ok) - fmt.Println(v) - // Output: - // true - // ISO-8859-1 value with unicode literal ⌘ and umlaut ä -} - -func ExampleLoad_utf8() { - p, _ := Load([]byte("key = UTF-8 value with unicode character ⌘ and umlaut ä"), UTF8) - v, ok := p.Get("key") - fmt.Println(ok) - fmt.Println(v) - // Output: - // true - // UTF-8 value with unicode character ⌘ and umlaut ä -} - -func ExampleProperties_GetBool() { - var input = ` - key=1 - key2=On - key3=YES - key4=true` - p, _ := Load([]byte(input), ISO_8859_1) - fmt.Println(p.GetBool("key", false)) - fmt.Println(p.GetBool("key2", false)) - fmt.Println(p.GetBool("key3", false)) - fmt.Println(p.GetBool("key4", false)) - fmt.Println(p.GetBool("keyX", false)) - // Output: - // true - // true - // true - // true - // false -} - -func ExampleProperties_GetString() { - p, _ := Load([]byte("key=value"), ISO_8859_1) - v := p.GetString("another key", "default value") - fmt.Println(v) - // Output: - // default value -} - -func Example() { - // Decode some key/value pairs with expressions - p, err := Load([]byte("key=value\nkey2=${key}"), ISO_8859_1) - if err != nil { - log.Fatal(err) - } - - // Get a valid key - if v, ok := p.Get("key"); ok { - fmt.Println(v) - } - - // Get an invalid key - if _, ok := p.Get("does not exist"); !ok { - fmt.Println("invalid key") - } - - // Get a key with a default value - v := p.GetString("does not exist", "some value") - fmt.Println(v) - - // Dump the expanded key/value pairs of the Properties - fmt.Println("Expanded key/value pairs") - fmt.Println(p) - - // Output: - // value - // invalid key - // some value - // Expanded key/value pairs - // key = value - // key2 = value -} diff --git a/vendor/github.com/magiconair/properties/integrate.go b/vendor/github.com/magiconair/properties/integrate.go index 1e972907..0d775e03 100644 --- a/vendor/github.com/magiconair/properties/integrate.go +++ b/vendor/github.com/magiconair/properties/integrate.go @@ -1,3 +1,7 @@ +// Copyright 2017 Frank Schroeder. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package properties import "flag" diff --git a/vendor/github.com/magiconair/properties/integrate_test.go b/vendor/github.com/magiconair/properties/integrate_test.go deleted file mode 100644 index 90daaa2c..00000000 --- a/vendor/github.com/magiconair/properties/integrate_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package properties - -import ( - "flag" - "fmt" - "testing" -) - -// TestFlag verifies Properties.MustFlag without flag.FlagSet.Parse -func TestFlag(t *testing.T) { - f := flag.NewFlagSet("src", flag.PanicOnError) - gotS := f.String("s", "?", "string flag") - gotI := f.Int("i", -1, "int flag") - - p := NewProperties() - p.Set("s", "t") - p.Set("i", "9") - p.MustFlag(f) - - if want := "t"; *gotS != want { - t.Errorf("Got string s=%q, want %q", *gotS, want) - } - if want := 9; *gotI != want { - t.Errorf("Got int i=%d, want %d", *gotI, want) - } -} - -// TestFlagOverride verifies Properties.MustFlag with flag.FlagSet.Parse. -func TestFlagOverride(t *testing.T) { - f := flag.NewFlagSet("src", flag.PanicOnError) - gotA := f.Int("a", 1, "remain default") - gotB := f.Int("b", 2, "customized") - gotC := f.Int("c", 3, "overridden") - - f.Parse([]string{"-c", "4"}) - - p := NewProperties() - p.Set("b", "5") - p.Set("c", "6") - p.MustFlag(f) - - if want := 1; *gotA != want { - t.Errorf("Got remain default a=%d, want %d", *gotA, want) - } - if want := 5; *gotB != want { - t.Errorf("Got customized b=%d, want %d", *gotB, want) - } - if want := 4; *gotC != want { - t.Errorf("Got overriden c=%d, want %d", *gotC, want) - } -} - -func ExampleProperties_MustFlag() { - x := flag.Int("x", 0, "demo customize") - y := flag.Int("y", 0, "demo override") - - // Demo alternative for flag.Parse(): - flag.CommandLine.Parse([]string{"-y", "10"}) - fmt.Printf("flagged as x=%d, y=%d\n", *x, *y) - - p := NewProperties() - p.Set("x", "7") - p.Set("y", "42") // note discard - p.MustFlag(flag.CommandLine) - fmt.Printf("configured to x=%d, y=%d\n", *x, *y) - - // Output: - // flagged as x=0, y=10 - // configured to x=7, y=10 -} diff --git a/vendor/github.com/magiconair/properties/lex.go b/vendor/github.com/magiconair/properties/lex.go index 1ae7a45d..a3cba031 100644 --- a/vendor/github.com/magiconair/properties/lex.go +++ b/vendor/github.com/magiconair/properties/lex.go @@ -1,4 +1,4 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. +// Copyright 2017 Frank Schroeder. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // @@ -72,7 +72,7 @@ type lexer struct { // next returns the next rune in the input. func (l *lexer) next() rune { - if int(l.pos) >= len(l.input) { + if l.pos >= len(l.input) { l.width = 0 return eof } @@ -96,8 +96,8 @@ func (l *lexer) backup() { // emit passes an item back to the client. func (l *lexer) emit(t itemType) { - item := item{t, l.start, string(l.runes)} - l.items <- item + i := item{t, l.start, string(l.runes)} + l.items <- i l.start = l.pos l.runes = l.runes[:0] } @@ -114,7 +114,7 @@ func (l *lexer) appendRune(r rune) { // accept consumes the next rune if it's from the valid set. func (l *lexer) accept(valid string) bool { - if strings.IndexRune(valid, l.next()) >= 0 { + if strings.ContainsRune(valid, l.next()) { return true } l.backup() @@ -123,7 +123,7 @@ func (l *lexer) accept(valid string) bool { // acceptRun consumes a run of runes from the valid set. func (l *lexer) acceptRun(valid string) { - for strings.IndexRune(valid, l.next()) >= 0 { + for strings.ContainsRune(valid, l.next()) { } l.backup() } @@ -156,9 +156,9 @@ func (l *lexer) errorf(format string, args ...interface{}) stateFn { // nextItem returns the next item from the input. func (l *lexer) nextItem() item { - item := <-l.items - l.lastPos = item.pos - return item + i := <-l.items + l.lastPos = i.pos + return i } // lex creates a new scanner for the input string. @@ -279,8 +279,7 @@ func lexValue(l *lexer) stateFn { for { switch r := l.next(); { case isEscape(r): - r := l.peek() - if isEOL(r) { + if isEOL(l.peek()) { l.next() l.acceptRun(whitespace) } else { diff --git a/vendor/github.com/magiconair/properties/load.go b/vendor/github.com/magiconair/properties/load.go index 431d4622..278cc2ea 100644 --- a/vendor/github.com/magiconair/properties/load.go +++ b/vendor/github.com/magiconair/properties/load.go @@ -1,4 +1,4 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. +// Copyright 2017 Frank Schroeder. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,7 +7,9 @@ package properties import ( "fmt" "io/ioutil" + "net/http" "os" + "strings" ) // Encoding specifies encoding of the input data. @@ -26,82 +28,197 @@ func Load(buf []byte, enc Encoding) (*Properties, error) { return loadBuf(buf, enc) } +// LoadString reads an UTF8 string into a properties struct. +func LoadString(s string) (*Properties, error) { + return loadBuf([]byte(s), UTF8) +} + +// LoadMap creates a new Properties struct from a string map. +func LoadMap(m map[string]string) *Properties { + p := NewProperties() + for k, v := range m { + p.Set(k, v) + } + return p +} + // LoadFile reads a file into a Properties struct. func LoadFile(filename string, enc Encoding) (*Properties, error) { - return loadFiles([]string{filename}, enc, false) + return loadAll([]string{filename}, enc, false) } // LoadFiles reads multiple files in the given order into // a Properties struct. If 'ignoreMissing' is true then // non-existent files will not be reported as error. func LoadFiles(filenames []string, enc Encoding, ignoreMissing bool) (*Properties, error) { - return loadFiles(filenames, enc, ignoreMissing) + return loadAll(filenames, enc, ignoreMissing) +} + +// LoadURL reads the content of the URL into a Properties struct. +// +// The encoding is determined via the Content-Type header which +// should be set to 'text/plain'. If the 'charset' parameter is +// missing, 'iso-8859-1' or 'latin1' the encoding is set to +// ISO-8859-1. If the 'charset' parameter is set to 'utf-8' the +// encoding is set to UTF-8. A missing content type header is +// interpreted as 'text/plain; charset=utf-8'. +func LoadURL(url string) (*Properties, error) { + return loadAll([]string{url}, UTF8, false) +} + +// LoadURLs reads the content of multiple URLs in the given order into a +// Properties struct. If 'ignoreMissing' is true then a 404 status code will +// not be reported as error. See LoadURL for the Content-Type header +// and the encoding. +func LoadURLs(urls []string, ignoreMissing bool) (*Properties, error) { + return loadAll(urls, UTF8, ignoreMissing) +} + +// LoadAll reads the content of multiple URLs or files in the given order into a +// Properties struct. If 'ignoreMissing' is true then a 404 status code or missing file will +// not be reported as error. Encoding sets the encoding for files. For the URLs please see +// LoadURL for the Content-Type header and the encoding. +func LoadAll(names []string, enc Encoding, ignoreMissing bool) (*Properties, error) { + return loadAll(names, enc, ignoreMissing) +} + +// MustLoadString reads an UTF8 string into a Properties struct and +// panics on error. +func MustLoadString(s string) *Properties { + return must(LoadString(s)) } // MustLoadFile reads a file into a Properties struct and // panics on error. func MustLoadFile(filename string, enc Encoding) *Properties { - return mustLoadFiles([]string{filename}, enc, false) + return must(LoadFile(filename, enc)) } // MustLoadFiles reads multiple files in the given order into // a Properties struct and panics on error. If 'ignoreMissing' // is true then non-existent files will not be reported as error. func MustLoadFiles(filenames []string, enc Encoding, ignoreMissing bool) *Properties { - return mustLoadFiles(filenames, enc, ignoreMissing) + return must(LoadFiles(filenames, enc, ignoreMissing)) } -// ---------------------------------------------------------------------------- +// MustLoadURL reads the content of a URL into a Properties struct and +// panics on error. +func MustLoadURL(url string) *Properties { + return must(LoadURL(url)) +} + +// MustLoadURLs reads the content of multiple URLs in the given order into a +// Properties struct and panics on error. If 'ignoreMissing' is true then a 404 +// status code will not be reported as error. +func MustLoadURLs(urls []string, ignoreMissing bool) *Properties { + return must(LoadURLs(urls, ignoreMissing)) +} + +// MustLoadAll reads the content of multiple URLs or files in the given order into a +// Properties struct. If 'ignoreMissing' is true then a 404 status code or missing file will +// not be reported as error. Encoding sets the encoding for files. For the URLs please see +// LoadURL for the Content-Type header and the encoding. It panics on error. +func MustLoadAll(names []string, enc Encoding, ignoreMissing bool) *Properties { + return must(LoadAll(names, enc, ignoreMissing)) +} func loadBuf(buf []byte, enc Encoding) (*Properties, error) { p, err := parse(convert(buf, enc)) if err != nil { return nil, err } - return p, p.check() } -func loadFiles(filenames []string, enc Encoding, ignoreMissing bool) (*Properties, error) { - buff := make([]byte, 0, 4096) - - for _, filename := range filenames { - f, err := expandFilename(filename) +func loadAll(names []string, enc Encoding, ignoreMissing bool) (*Properties, error) { + result := NewProperties() + for _, name := range names { + n, err := expandName(name) if err != nil { return nil, err } - - buf, err := ioutil.ReadFile(f) + var p *Properties + if strings.HasPrefix(n, "http://") || strings.HasPrefix(n, "https://") { + p, err = loadURL(n, ignoreMissing) + } else { + p, err = loadFile(n, enc, ignoreMissing) + } if err != nil { - if ignoreMissing && os.IsNotExist(err) { - // TODO(frank): should we log that we are skipping the file? - continue - } return nil, err } + result.Merge(p) - // concatenate the buffers and add a new line in case - // the previous file didn't end with a new line - buff = append(append(buff, buf...), '\n') } - - return loadBuf(buff, enc) + return result, result.check() } -func mustLoadFiles(filenames []string, enc Encoding, ignoreMissing bool) *Properties { - p, err := loadFiles(filenames, enc, ignoreMissing) +func loadFile(filename string, enc Encoding, ignoreMissing bool) (*Properties, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + if ignoreMissing && os.IsNotExist(err) { + LogPrintf("properties: %s not found. skipping", filename) + return NewProperties(), nil + } + return nil, err + } + p, err := parse(convert(data, enc)) + if err != nil { + return nil, err + } + return p, nil +} + +func loadURL(url string, ignoreMissing bool) (*Properties, error) { + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("properties: error fetching %q. %s", url, err) + } + if resp.StatusCode == 404 && ignoreMissing { + LogPrintf("properties: %s returned %d. skipping", url, resp.StatusCode) + return NewProperties(), nil + } + if resp.StatusCode != 200 { + return nil, fmt.Errorf("properties: %s returned %d", url, resp.StatusCode) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("properties: %s error reading response. %s", url, err) + } + if err = resp.Body.Close(); err != nil { + return nil, fmt.Errorf("properties: %s error reading response. %s", url, err) + } + + ct := resp.Header.Get("Content-Type") + var enc Encoding + switch strings.ToLower(ct) { + case "text/plain", "text/plain; charset=iso-8859-1", "text/plain; charset=latin1": + enc = ISO_8859_1 + case "", "text/plain; charset=utf-8": + enc = UTF8 + default: + return nil, fmt.Errorf("properties: invalid content type %s", ct) + } + + p, err := parse(convert(body, enc)) + if err != nil { + return nil, err + } + return p, nil +} + +func must(p *Properties, err error) *Properties { if err != nil { ErrorHandler(err) } return p } -// expandFilename expands ${ENV_VAR} expressions in a filename. +// expandName expands ${ENV_VAR} expressions in a name. // If the environment variable does not exist then it will be replaced // with an empty string. Malformed expressions like "${ENV_VAR" will // be reported as error. -func expandFilename(filename string) (string, error) { - return expand(filename, make(map[string]bool), "${", "}", make(map[string]string)) +func expandName(name string) (string, error) { + return expand(name, make(map[string]bool), "${", "}", make(map[string]string)) } // Interprets a byte buffer either as an ISO-8859-1 or UTF-8 encoded string. diff --git a/vendor/github.com/magiconair/properties/load_test.go b/vendor/github.com/magiconair/properties/load_test.go deleted file mode 100644 index e6dae8db..00000000 --- a/vendor/github.com/magiconair/properties/load_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package properties - -import ( - "fmt" - "io/ioutil" - "os" - "strings" - - . "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" -) - -type LoadSuite struct { - tempFiles []string -} - -var ( - _ = Suite(&LoadSuite{}) -) - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TestLoadFailsWithNotExistingFile(c *C) { - _, err := LoadFile("doesnotexist.properties", ISO_8859_1) - c.Assert(err, NotNil) - c.Assert(err, ErrorMatches, "open.*no such file or directory") -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TestLoadFilesFailsOnNotExistingFile(c *C) { - _, err := LoadFiles([]string{"doesnotexist.properties"}, ISO_8859_1, false) - c.Assert(err, NotNil) - c.Assert(err, ErrorMatches, "open.*no such file or directory") -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TestLoadFilesDoesNotFailOnNotExistingFileAndIgnoreMissing(c *C) { - p, err := LoadFiles([]string{"doesnotexist.properties"}, ISO_8859_1, true) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 0) -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TestLoad(c *C) { - filename := s.makeFile(c, "key=value") - p := MustLoadFile(filename, ISO_8859_1) - - c.Assert(p.Len(), Equals, 1) - assertKeyValues(c, "", p, "key", "value") -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TestLoadFiles(c *C) { - filename := s.makeFile(c, "key=value") - filename2 := s.makeFile(c, "key2=value2") - p := MustLoadFiles([]string{filename, filename2}, ISO_8859_1, false) - assertKeyValues(c, "", p, "key", "value", "key2", "value2") -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TestLoadExpandedFile(c *C) { - filename := s.makeFilePrefix(c, os.Getenv("USER"), "key=value") - filename = strings.Replace(filename, os.Getenv("USER"), "${USER}", -1) - p := MustLoadFile(filename, ISO_8859_1) - assertKeyValues(c, "", p, "key", "value") -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TestLoadFilesAndIgnoreMissing(c *C) { - filename := s.makeFile(c, "key=value") - filename2 := s.makeFile(c, "key2=value2") - p := MustLoadFiles([]string{filename, filename + "foo", filename2, filename2 + "foo"}, ISO_8859_1, true) - assertKeyValues(c, "", p, "key", "value", "key2", "value2") -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) SetUpSuite(c *C) { - s.tempFiles = make([]string, 0) -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) TearDownSuite(c *C) { - for _, path := range s.tempFiles { - err := os.Remove(path) - if err != nil { - fmt.Printf("os.Remove: %v", err) - } - } -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) makeFile(c *C, data string) string { - return s.makeFilePrefix(c, "properties", data) -} - -// ---------------------------------------------------------------------------- - -func (s *LoadSuite) makeFilePrefix(c *C, prefix, data string) string { - f, err := ioutil.TempFile("", prefix) - if err != nil { - fmt.Printf("ioutil.TempFile: %v", err) - c.FailNow() - } - - // remember the temp file so that we can remove it later - s.tempFiles = append(s.tempFiles, f.Name()) - - n, err := fmt.Fprint(f, data) - if err != nil { - fmt.Printf("fmt.Fprintln: %v", err) - c.FailNow() - } - if n != len(data) { - fmt.Printf("Data size mismatch. expected=%d wrote=%d\n", len(data), n) - c.FailNow() - } - - err = f.Close() - if err != nil { - fmt.Printf("f.Close: %v", err) - c.FailNow() - } - - return f.Name() -} diff --git a/vendor/github.com/magiconair/properties/parser.go b/vendor/github.com/magiconair/properties/parser.go index bb71fb94..90f555cb 100644 --- a/vendor/github.com/magiconair/properties/parser.go +++ b/vendor/github.com/magiconair/properties/parser.go @@ -1,4 +1,4 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. +// Copyright 2017 Frank Schroeder. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/vendor/github.com/magiconair/properties/properties.go b/vendor/github.com/magiconair/properties/properties.go index bd5cba10..4f3d5a45 100644 --- a/vendor/github.com/magiconair/properties/properties.go +++ b/vendor/github.com/magiconair/properties/properties.go @@ -1,4 +1,4 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. +// Copyright 2017 Frank Schroeder. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -26,7 +26,13 @@ type ErrorHandlerFunc func(error) // ErrorHandler is the function which handles failures of the MustXXX() // functions. The default is LogFatalHandler. -var ErrorHandler = LogFatalHandler +var ErrorHandler ErrorHandlerFunc = LogFatalHandler + +// LogHandlerFunc defines the function prototype for logging errors. +type LogHandlerFunc func(fmt string, args ...interface{}) + +// LogPrintf defines a log handler which uses log.Printf. +var LogPrintf LogHandlerFunc = log.Printf // LogFatalHandler handles the error by logging a fatal error and exiting. func LogFatalHandler(err error) { @@ -440,6 +446,8 @@ func (p *Properties) FilterRegexp(re *regexp.Regexp) *Properties { pp := NewProperties() for _, k := range p.k { if re.MatchString(k) { + // TODO(fs): we are ignoring the error which flags a circular reference. + // TODO(fs): since we are just copying a subset of keys this cannot happen (fingers crossed) pp.Set(k, p.m[k]) } } @@ -452,6 +460,8 @@ func (p *Properties) FilterPrefix(prefix string) *Properties { pp := NewProperties() for _, k := range p.k { if strings.HasPrefix(k, prefix) { + // TODO(fs): we are ignoring the error which flags a circular reference. + // TODO(fs): since we are just copying a subset of keys this cannot happen (fingers crossed) pp.Set(k, p.m[k]) } } @@ -465,6 +475,9 @@ func (p *Properties) FilterStripPrefix(prefix string) *Properties { n := len(prefix) for _, k := range p.k { if len(k) > len(prefix) && strings.HasPrefix(k, prefix) { + // TODO(fs): we are ignoring the error which flags a circular reference. + // TODO(fs): since we are modifying keys I am not entirely sure whether we can create a circular reference + // TODO(fs): this function should probably return an error but the signature is fixed pp.Set(k[n:], p.m[k]) } } @@ -479,9 +492,7 @@ func (p *Properties) Len() int { // Keys returns all keys in the same order as in the input. func (p *Properties) Keys() []string { keys := make([]string, len(p.k)) - for i, k := range p.k { - keys[i] = k - } + copy(keys, p.k) return keys } @@ -531,6 +542,13 @@ func (p *Properties) Set(key, value string) (prev string, ok bool, err error) { return prev, ok, nil } +// SetValue sets property key to the default string value +// as defined by fmt.Sprintf("%v"). +func (p *Properties) SetValue(key string, value interface{}) error { + _, _, err := p.Set(key, fmt.Sprintf("%v", value)) + return err +} + // MustSet sets the property key to the corresponding value. // If a value for key existed before then ok is true and prev // contains the previous value. An empty key is silently ignored. @@ -611,6 +629,30 @@ func (p *Properties) WriteComment(w io.Writer, prefix string, enc Encoding) (n i return } +// Map returns a copy of the properties as a map. +func (p *Properties) Map() map[string]string { + m := make(map[string]string) + for k, v := range p.m { + m[k] = v + } + return m +} + +// FilterFunc returns a copy of the properties which includes the values which passed all filters. +func (p *Properties) FilterFunc(filters ...func(k, v string) bool) *Properties { + pp := NewProperties() +outer: + for k, v := range p.m { + for _, f := range filters { + if !f(k, v) { + continue outer + } + pp.Set(k, v) + } + } + return pp +} + // ---------------------------------------------------------------------------- // Delete removes the key and its comments. @@ -620,12 +662,32 @@ func (p *Properties) Delete(key string) { newKeys := []string{} for _, k := range p.k { if k != key { - newKeys = append(newKeys, key) + newKeys = append(newKeys, k) } } p.k = newKeys } +// Merge merges properties, comments and keys from other *Properties into p +func (p *Properties) Merge(other *Properties) { + for k, v := range other.m { + p.m[k] = v + } + for k, v := range other.c { + p.c[k] = v + } + +outer: + for _, otherKey := range other.k { + for _, key := range p.k { + if otherKey == key { + continue outer + } + } + p.k = append(p.k, otherKey) + } +} + // ---------------------------------------------------------------------------- // check expands all values and returns an error if a circular reference or diff --git a/vendor/github.com/magiconair/properties/properties_test.go b/vendor/github.com/magiconair/properties/properties_test.go deleted file mode 100644 index dc743855..00000000 --- a/vendor/github.com/magiconair/properties/properties_test.go +++ /dev/null @@ -1,906 +0,0 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package properties - -import ( - "bytes" - "flag" - "fmt" - "math" - "os" - "strings" - "testing" - "time" - - . "github.com/magiconair/properties/_third_party/gopkg.in/check.v1" -) - -func Test(t *testing.T) { TestingT(t) } - -type TestSuite struct { - prevHandler ErrorHandlerFunc -} - -var ( - _ = Suite(&TestSuite{}) - verbose = flag.Bool("verbose", false, "Verbose output") -) - -// -------------------------------------------------------------------- - -func (s *TestSuite) SetUpSuite(c *C) { - s.prevHandler = ErrorHandler - ErrorHandler = PanicHandler -} - -// -------------------------------------------------------------------- - -func (s *TestSuite) TearDownSuite(c *C) { - ErrorHandler = s.prevHandler -} - -// ---------------------------------------------------------------------------- - -// define test cases in the form of -// {"input", "key1", "value1", "key2", "value2", ...} -var complexTests = [][]string{ - // whitespace prefix - {" key=value", "key", "value"}, // SPACE prefix - {"\fkey=value", "key", "value"}, // FF prefix - {"\tkey=value", "key", "value"}, // TAB prefix - {" \f\tkey=value", "key", "value"}, // mix prefix - - // multiple keys - {"key1=value1\nkey2=value2\n", "key1", "value1", "key2", "value2"}, - {"key1=value1\rkey2=value2\r", "key1", "value1", "key2", "value2"}, - {"key1=value1\r\nkey2=value2\r\n", "key1", "value1", "key2", "value2"}, - - // blank lines - {"\nkey=value\n", "key", "value"}, - {"\rkey=value\r", "key", "value"}, - {"\r\nkey=value\r\n", "key", "value"}, - - // escaped chars in key - {"k\\ ey = value", "k ey", "value"}, - {"k\\:ey = value", "k:ey", "value"}, - {"k\\=ey = value", "k=ey", "value"}, - {"k\\fey = value", "k\fey", "value"}, - {"k\\ney = value", "k\ney", "value"}, - {"k\\rey = value", "k\rey", "value"}, - {"k\\tey = value", "k\tey", "value"}, - - // escaped chars in value - {"key = v\\ alue", "key", "v alue"}, - {"key = v\\:alue", "key", "v:alue"}, - {"key = v\\=alue", "key", "v=alue"}, - {"key = v\\falue", "key", "v\falue"}, - {"key = v\\nalue", "key", "v\nalue"}, - {"key = v\\ralue", "key", "v\ralue"}, - {"key = v\\talue", "key", "v\talue"}, - - // silently dropped escape character - {"k\\zey = value", "kzey", "value"}, - {"key = v\\zalue", "key", "vzalue"}, - - // unicode literals - {"key\\u2318 = value", "key⌘", "value"}, - {"k\\u2318ey = value", "k⌘ey", "value"}, - {"key = value\\u2318", "key", "value⌘"}, - {"key = valu\\u2318e", "key", "valu⌘e"}, - - // multiline values - {"key = valueA,\\\n valueB", "key", "valueA,valueB"}, // SPACE indent - {"key = valueA,\\\n\f\f\fvalueB", "key", "valueA,valueB"}, // FF indent - {"key = valueA,\\\n\t\t\tvalueB", "key", "valueA,valueB"}, // TAB indent - {"key = valueA,\\\n \f\tvalueB", "key", "valueA,valueB"}, // mix indent - - // comments - {"# this is a comment\n! and so is this\nkey1=value1\nkey#2=value#2\n\nkey!3=value!3\n# and another one\n! and the final one", "key1", "value1", "key#2", "value#2", "key!3", "value!3"}, - - // expansion tests - {"key=value\nkey2=${key}", "key", "value", "key2", "value"}, - {"key=value\nkey2=aa${key}", "key", "value", "key2", "aavalue"}, - {"key=value\nkey2=${key}bb", "key", "value", "key2", "valuebb"}, - {"key=value\nkey2=aa${key}bb", "key", "value", "key2", "aavaluebb"}, - {"key=value\nkey2=${key}\nkey3=${key2}", "key", "value", "key2", "value", "key3", "value"}, - {"key=${USER}", "key", os.Getenv("USER")}, - {"key=${USER}\nUSER=value", "key", "value", "USER", "value"}, -} - -// ---------------------------------------------------------------------------- - -var commentTests = []struct { - input, key, value string - comments []string -}{ - {"key=value", "key", "value", nil}, - {"#\nkey=value", "key", "value", []string{""}}, - {"#comment\nkey=value", "key", "value", []string{"comment"}}, - {"# comment\nkey=value", "key", "value", []string{"comment"}}, - {"# comment\nkey=value", "key", "value", []string{"comment"}}, - {"# comment\n\nkey=value", "key", "value", []string{"comment"}}, - {"# comment1\n# comment2\nkey=value", "key", "value", []string{"comment1", "comment2"}}, - {"# comment1\n\n# comment2\n\nkey=value", "key", "value", []string{"comment1", "comment2"}}, - {"!comment\nkey=value", "key", "value", []string{"comment"}}, - {"! comment\nkey=value", "key", "value", []string{"comment"}}, - {"! comment\nkey=value", "key", "value", []string{"comment"}}, - {"! comment\n\nkey=value", "key", "value", []string{"comment"}}, - {"! comment1\n! comment2\nkey=value", "key", "value", []string{"comment1", "comment2"}}, - {"! comment1\n\n! comment2\n\nkey=value", "key", "value", []string{"comment1", "comment2"}}, -} - -// ---------------------------------------------------------------------------- - -var errorTests = []struct { - input, msg string -}{ - // unicode literals - {"key\\u1 = value", "invalid unicode literal"}, - {"key\\u12 = value", "invalid unicode literal"}, - {"key\\u123 = value", "invalid unicode literal"}, - {"key\\u123g = value", "invalid unicode literal"}, - {"key\\u123", "invalid unicode literal"}, - - // circular references - {"key=${key}", "circular reference"}, - {"key1=${key2}\nkey2=${key1}", "circular reference"}, - - // malformed expressions - {"key=${ke", "malformed expression"}, - {"key=valu${ke", "malformed expression"}, -} - -// ---------------------------------------------------------------------------- - -var writeTests = []struct { - input, output, encoding string -}{ - // ISO-8859-1 tests - {"key = value", "key = value\n", "ISO-8859-1"}, - {"key = value \\\n continued", "key = value continued\n", "ISO-8859-1"}, - {"key⌘ = value", "key\\u2318 = value\n", "ISO-8859-1"}, - {"ke\\ \\:y = value", "ke\\ \\:y = value\n", "ISO-8859-1"}, - - // UTF-8 tests - {"key = value", "key = value\n", "UTF-8"}, - {"key = value \\\n continued", "key = value continued\n", "UTF-8"}, - {"key⌘ = value⌘", "key⌘ = value⌘\n", "UTF-8"}, - {"ke\\ \\:y = value", "ke\\ \\:y = value\n", "UTF-8"}, -} - -// ---------------------------------------------------------------------------- - -var writeCommentTests = []struct { - input, output, encoding string -}{ - // ISO-8859-1 tests - {"key = value", "key = value\n", "ISO-8859-1"}, - {"#\nkey = value", "key = value\n", "ISO-8859-1"}, - {"#\n#\n#\nkey = value", "key = value\n", "ISO-8859-1"}, - {"# comment\nkey = value", "# comment\nkey = value\n", "ISO-8859-1"}, - {"\n# comment\nkey = value", "# comment\nkey = value\n", "ISO-8859-1"}, - {"# comment\n\nkey = value", "# comment\nkey = value\n", "ISO-8859-1"}, - {"# comment1\n# comment2\nkey = value", "# comment1\n# comment2\nkey = value\n", "ISO-8859-1"}, - {"#comment1\nkey1 = value1\n#comment2\nkey2 = value2", "# comment1\nkey1 = value1\n\n# comment2\nkey2 = value2\n", "ISO-8859-1"}, - - // UTF-8 tests - {"key = value", "key = value\n", "UTF-8"}, - {"# comment⌘\nkey = value⌘", "# comment⌘\nkey = value⌘\n", "UTF-8"}, - {"\n# comment⌘\nkey = value⌘", "# comment⌘\nkey = value⌘\n", "UTF-8"}, - {"# comment⌘\n\nkey = value⌘", "# comment⌘\nkey = value⌘\n", "UTF-8"}, - {"# comment1⌘\n# comment2⌘\nkey = value⌘", "# comment1⌘\n# comment2⌘\nkey = value⌘\n", "UTF-8"}, - {"#comment1⌘\nkey1 = value1⌘\n#comment2⌘\nkey2 = value2⌘", "# comment1⌘\nkey1 = value1⌘\n\n# comment2⌘\nkey2 = value2⌘\n", "UTF-8"}, -} - -// ---------------------------------------------------------------------------- - -var boolTests = []struct { - input, key string - def, value bool -}{ - // valid values for TRUE - {"key = 1", "key", false, true}, - {"key = on", "key", false, true}, - {"key = On", "key", false, true}, - {"key = ON", "key", false, true}, - {"key = true", "key", false, true}, - {"key = True", "key", false, true}, - {"key = TRUE", "key", false, true}, - {"key = yes", "key", false, true}, - {"key = Yes", "key", false, true}, - {"key = YES", "key", false, true}, - - // valid values for FALSE (all other) - {"key = 0", "key", true, false}, - {"key = off", "key", true, false}, - {"key = false", "key", true, false}, - {"key = no", "key", true, false}, - - // non existent key - {"key = true", "key2", false, false}, -} - -// ---------------------------------------------------------------------------- - -var durationTests = []struct { - input, key string - def, value time.Duration -}{ - // valid values - {"key = 1", "key", 999, 1}, - {"key = 0", "key", 999, 0}, - {"key = -1", "key", 999, -1}, - {"key = 0123", "key", 999, 123}, - - // invalid values - {"key = 0xff", "key", 999, 999}, - {"key = 1.0", "key", 999, 999}, - {"key = a", "key", 999, 999}, - - // non existent key - {"key = 1", "key2", 999, 999}, -} - -// ---------------------------------------------------------------------------- - -var parsedDurationTests = []struct { - input, key string - def, value time.Duration -}{ - // valid values - {"key = -1ns", "key", 999, -1 * time.Nanosecond}, - {"key = 300ms", "key", 999, 300 * time.Millisecond}, - {"key = 5s", "key", 999, 5 * time.Second}, - {"key = 3h", "key", 999, 3 * time.Hour}, - {"key = 2h45m", "key", 999, 2*time.Hour + 45*time.Minute}, - - // invalid values - {"key = 0xff", "key", 999, 999}, - {"key = 1.0", "key", 999, 999}, - {"key = a", "key", 999, 999}, - {"key = 1", "key", 999, 999}, - {"key = 0", "key", 999, 0}, - - // non existent key - {"key = 1", "key2", 999, 999}, -} - -// ---------------------------------------------------------------------------- - -var floatTests = []struct { - input, key string - def, value float64 -}{ - // valid values - {"key = 1.0", "key", 999, 1.0}, - {"key = 0.0", "key", 999, 0.0}, - {"key = -1.0", "key", 999, -1.0}, - {"key = 1", "key", 999, 1}, - {"key = 0", "key", 999, 0}, - {"key = -1", "key", 999, -1}, - {"key = 0123", "key", 999, 123}, - - // invalid values - {"key = 0xff", "key", 999, 999}, - {"key = a", "key", 999, 999}, - - // non existent key - {"key = 1", "key2", 999, 999}, -} - -// ---------------------------------------------------------------------------- - -var int64Tests = []struct { - input, key string - def, value int64 -}{ - // valid values - {"key = 1", "key", 999, 1}, - {"key = 0", "key", 999, 0}, - {"key = -1", "key", 999, -1}, - {"key = 0123", "key", 999, 123}, - - // invalid values - {"key = 0xff", "key", 999, 999}, - {"key = 1.0", "key", 999, 999}, - {"key = a", "key", 999, 999}, - - // non existent key - {"key = 1", "key2", 999, 999}, -} - -// ---------------------------------------------------------------------------- - -var uint64Tests = []struct { - input, key string - def, value uint64 -}{ - // valid values - {"key = 1", "key", 999, 1}, - {"key = 0", "key", 999, 0}, - {"key = 0123", "key", 999, 123}, - - // invalid values - {"key = -1", "key", 999, 999}, - {"key = 0xff", "key", 999, 999}, - {"key = 1.0", "key", 999, 999}, - {"key = a", "key", 999, 999}, - - // non existent key - {"key = 1", "key2", 999, 999}, -} - -// ---------------------------------------------------------------------------- - -var stringTests = []struct { - input, key string - def, value string -}{ - // valid values - {"key = abc", "key", "def", "abc"}, - - // non existent key - {"key = abc", "key2", "def", "def"}, -} - -// ---------------------------------------------------------------------------- - -var keysTests = []struct { - input string - keys []string -}{ - {"", []string{}}, - {"key = abc", []string{"key"}}, - {"key = abc\nkey2=def", []string{"key", "key2"}}, - {"key2 = abc\nkey=def", []string{"key2", "key"}}, - {"key = abc\nkey=def", []string{"key"}}, -} - -// ---------------------------------------------------------------------------- - -var filterTests = []struct { - input string - pattern string - keys []string - err string -}{ - {"", "", []string{}, ""}, - {"", "abc", []string{}, ""}, - {"key=value", "", []string{"key"}, ""}, - {"key=value", "key=", []string{}, ""}, - {"key=value\nfoo=bar", "", []string{"foo", "key"}, ""}, - {"key=value\nfoo=bar", "f", []string{"foo"}, ""}, - {"key=value\nfoo=bar", "fo", []string{"foo"}, ""}, - {"key=value\nfoo=bar", "foo", []string{"foo"}, ""}, - {"key=value\nfoo=bar", "fooo", []string{}, ""}, - {"key=value\nkey2=value2\nfoo=bar", "ey", []string{"key", "key2"}, ""}, - {"key=value\nkey2=value2\nfoo=bar", "key", []string{"key", "key2"}, ""}, - {"key=value\nkey2=value2\nfoo=bar", "^key", []string{"key", "key2"}, ""}, - {"key=value\nkey2=value2\nfoo=bar", "^(key|foo)", []string{"foo", "key", "key2"}, ""}, - {"key=value\nkey2=value2\nfoo=bar", "[ abc", nil, "error parsing regexp.*"}, -} - -// ---------------------------------------------------------------------------- - -var filterPrefixTests = []struct { - input string - prefix string - keys []string -}{ - {"", "", []string{}}, - {"", "abc", []string{}}, - {"key=value", "", []string{"key"}}, - {"key=value", "key=", []string{}}, - {"key=value\nfoo=bar", "", []string{"foo", "key"}}, - {"key=value\nfoo=bar", "f", []string{"foo"}}, - {"key=value\nfoo=bar", "fo", []string{"foo"}}, - {"key=value\nfoo=bar", "foo", []string{"foo"}}, - {"key=value\nfoo=bar", "fooo", []string{}}, - {"key=value\nkey2=value2\nfoo=bar", "key", []string{"key", "key2"}}, -} - -// ---------------------------------------------------------------------------- - -var filterStripPrefixTests = []struct { - input string - prefix string - keys []string -}{ - {"", "", []string{}}, - {"", "abc", []string{}}, - {"key=value", "", []string{"key"}}, - {"key=value", "key=", []string{}}, - {"key=value\nfoo=bar", "", []string{"foo", "key"}}, - {"key=value\nfoo=bar", "f", []string{"foo"}}, - {"key=value\nfoo=bar", "fo", []string{"foo"}}, - {"key=value\nfoo=bar", "foo", []string{"foo"}}, - {"key=value\nfoo=bar", "fooo", []string{}}, - {"key=value\nkey2=value2\nfoo=bar", "key", []string{"key", "key2"}}, -} - -// ---------------------------------------------------------------------------- - -var setTests = []struct { - input string - key, value string - prev string - ok bool - err string - keys []string -}{ - {"", "", "", "", false, "", []string{}}, - {"", "key", "value", "", false, "", []string{"key"}}, - {"key=value", "key2", "value2", "", false, "", []string{"key", "key2"}}, - {"key=value", "abc", "value3", "", false, "", []string{"key", "abc"}}, - {"key=value", "key", "value3", "value", true, "", []string{"key"}}, -} - -// ---------------------------------------------------------------------------- - -// TestBasic tests basic single key/value combinations with all possible -// whitespace, delimiter and newline permutations. -func (s *TestSuite) TestBasic(c *C) { - testWhitespaceAndDelimiterCombinations(c, "key", "") - testWhitespaceAndDelimiterCombinations(c, "key", "value") - testWhitespaceAndDelimiterCombinations(c, "key", "value ") -} - -func (s *TestSuite) TestComplex(c *C) { - for _, test := range complexTests { - testKeyValue(c, test[0], test[1:]...) - } -} - -func (s *TestSuite) TestErrors(c *C) { - for _, test := range errorTests { - _, err := Load([]byte(test.input), ISO_8859_1) - c.Assert(err, NotNil) - c.Assert(strings.Contains(err.Error(), test.msg), Equals, true, Commentf("Expected %q got %q", test.msg, err.Error())) - } -} - -func (s *TestSuite) TestDisableExpansion(c *C) { - input := "key=value\nkey2=${key}" - p, err := parse(input) - p.DisableExpansion = true - c.Assert(err, IsNil) - c.Assert(p.MustGet("key"), Equals, "value") - c.Assert(p.MustGet("key2"), Equals, "${key}") - - // with expansion disabled we can introduce circular references - p.Set("keyA", "${keyB}") - p.Set("keyB", "${keyA}") - c.Assert(p.MustGet("keyA"), Equals, "${keyB}") - c.Assert(p.MustGet("keyB"), Equals, "${keyA}") -} - -func (s *TestSuite) TestMustGet(c *C) { - input := "key = value\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGet("key"), Equals, "value") - c.Assert(func() { p.MustGet("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetBool(c *C) { - for _, test := range boolTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetBool(test.key, test.def), Equals, test.value) - } -} - -func (s *TestSuite) TestMustGetBool(c *C) { - input := "key = true\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetBool("key"), Equals, true) - c.Assert(func() { p.MustGetBool("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetDuration(c *C) { - for _, test := range durationTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetDuration(test.key, test.def), Equals, test.value) - } -} - -func (s *TestSuite) TestMustGetDuration(c *C) { - input := "key = 123\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetDuration("key"), Equals, time.Duration(123)) - c.Assert(func() { p.MustGetDuration("key2") }, PanicMatches, "strconv.ParseInt: parsing.*") - c.Assert(func() { p.MustGetDuration("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetParsedDuration(c *C) { - for _, test := range parsedDurationTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetParsedDuration(test.key, test.def), Equals, test.value) - } -} - -func (s *TestSuite) TestMustGetParsedDuration(c *C) { - input := "key = 123ms\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetParsedDuration("key"), Equals, 123*time.Millisecond) - c.Assert(func() { p.MustGetParsedDuration("key2") }, PanicMatches, "time: invalid duration ghi") - c.Assert(func() { p.MustGetParsedDuration("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetFloat64(c *C) { - for _, test := range floatTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetFloat64(test.key, test.def), Equals, test.value) - } -} - -func (s *TestSuite) TestMustGetFloat64(c *C) { - input := "key = 123\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetFloat64("key"), Equals, float64(123)) - c.Assert(func() { p.MustGetFloat64("key2") }, PanicMatches, "strconv.ParseFloat: parsing.*") - c.Assert(func() { p.MustGetFloat64("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetInt(c *C) { - for _, test := range int64Tests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetInt(test.key, int(test.def)), Equals, int(test.value)) - } -} - -func (s *TestSuite) TestMustGetInt(c *C) { - input := "key = 123\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetInt("key"), Equals, int(123)) - c.Assert(func() { p.MustGetInt("key2") }, PanicMatches, "strconv.ParseInt: parsing.*") - c.Assert(func() { p.MustGetInt("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetInt64(c *C) { - for _, test := range int64Tests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetInt64(test.key, test.def), Equals, test.value) - } -} - -func (s *TestSuite) TestMustGetInt64(c *C) { - input := "key = 123\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetInt64("key"), Equals, int64(123)) - c.Assert(func() { p.MustGetInt64("key2") }, PanicMatches, "strconv.ParseInt: parsing.*") - c.Assert(func() { p.MustGetInt64("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetUint(c *C) { - for _, test := range uint64Tests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetUint(test.key, uint(test.def)), Equals, uint(test.value)) - } -} - -func (s *TestSuite) TestMustGetUint(c *C) { - input := "key = 123\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetUint("key"), Equals, uint(123)) - c.Assert(func() { p.MustGetUint64("key2") }, PanicMatches, "strconv.ParseUint: parsing.*") - c.Assert(func() { p.MustGetUint64("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetUint64(c *C) { - for _, test := range uint64Tests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetUint64(test.key, test.def), Equals, test.value) - } -} - -func (s *TestSuite) TestMustGetUint64(c *C) { - input := "key = 123\nkey2 = ghi" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetUint64("key"), Equals, uint64(123)) - c.Assert(func() { p.MustGetUint64("key2") }, PanicMatches, "strconv.ParseUint: parsing.*") - c.Assert(func() { p.MustGetUint64("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestGetString(c *C) { - for _, test := range stringTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, 1) - c.Assert(p.GetString(test.key, test.def), Equals, test.value) - } -} - -func (s *TestSuite) TestMustGetString(c *C) { - input := `key = value` - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetString("key"), Equals, "value") - c.Assert(func() { p.MustGetString("invalid") }, PanicMatches, "unknown property: invalid") -} - -func (s *TestSuite) TestComment(c *C) { - for _, test := range commentTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.MustGetString(test.key), Equals, test.value) - c.Assert(p.GetComments(test.key), DeepEquals, test.comments) - if test.comments != nil { - c.Assert(p.GetComment(test.key), Equals, test.comments[len(test.comments)-1]) - } else { - c.Assert(p.GetComment(test.key), Equals, "") - } - - // test setting comments - if len(test.comments) > 0 { - // set single comment - p.ClearComments() - c.Assert(len(p.c), Equals, 0) - p.SetComment(test.key, test.comments[0]) - c.Assert(p.GetComment(test.key), Equals, test.comments[0]) - - // set multiple comments - p.ClearComments() - c.Assert(len(p.c), Equals, 0) - p.SetComments(test.key, test.comments) - c.Assert(p.GetComments(test.key), DeepEquals, test.comments) - - // clear comments for a key - p.SetComments(test.key, nil) - c.Assert(p.GetComment(test.key), Equals, "") - c.Assert(p.GetComments(test.key), IsNil) - } - } -} - -func (s *TestSuite) TestFilter(c *C) { - for _, test := range filterTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - pp, err := p.Filter(test.pattern) - if err != nil { - c.Assert(err, ErrorMatches, test.err) - continue - } - c.Assert(pp, NotNil) - c.Assert(pp.Len(), Equals, len(test.keys)) - for _, key := range test.keys { - v1, ok1 := p.Get(key) - v2, ok2 := pp.Get(key) - c.Assert(ok1, Equals, true) - c.Assert(ok2, Equals, true) - c.Assert(v1, Equals, v2) - } - } -} - -func (s *TestSuite) TestFilterPrefix(c *C) { - for _, test := range filterPrefixTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - pp := p.FilterPrefix(test.prefix) - c.Assert(pp, NotNil) - c.Assert(pp.Len(), Equals, len(test.keys)) - for _, key := range test.keys { - v1, ok1 := p.Get(key) - v2, ok2 := pp.Get(key) - c.Assert(ok1, Equals, true) - c.Assert(ok2, Equals, true) - c.Assert(v1, Equals, v2) - } - } -} - -func (s *TestSuite) TestKeys(c *C) { - for _, test := range keysTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - c.Assert(p.Len(), Equals, len(test.keys)) - c.Assert(len(p.Keys()), Equals, len(test.keys)) - c.Assert(p.Keys(), DeepEquals, test.keys) - } -} - -func (s *TestSuite) TestSet(c *C) { - for _, test := range setTests { - p, err := parse(test.input) - c.Assert(err, IsNil) - prev, ok, err := p.Set(test.key, test.value) - if test.err != "" { - c.Assert(err, ErrorMatches, test.err) - continue - } - - c.Assert(err, IsNil) - c.Assert(ok, Equals, test.ok) - if ok { - c.Assert(prev, Equals, test.prev) - } - c.Assert(p.Keys(), DeepEquals, test.keys) - } -} - -func (s *TestSuite) TestMustSet(c *C) { - input := "key=${key}" - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(func() { p.MustSet("key", "${key}") }, PanicMatches, "circular reference .*") -} - -func (s *TestSuite) TestWrite(c *C) { - for _, test := range writeTests { - p, err := parse(test.input) - - buf := new(bytes.Buffer) - var n int - switch test.encoding { - case "UTF-8": - n, err = p.Write(buf, UTF8) - case "ISO-8859-1": - n, err = p.Write(buf, ISO_8859_1) - } - c.Assert(err, IsNil) - s := string(buf.Bytes()) - c.Assert(n, Equals, len(test.output), Commentf("input=%q expected=%q obtained=%q", test.input, test.output, s)) - c.Assert(s, Equals, test.output, Commentf("input=%q expected=%q obtained=%q", test.input, test.output, s)) - } -} - -func (s *TestSuite) TestWriteComment(c *C) { - for _, test := range writeCommentTests { - p, err := parse(test.input) - - buf := new(bytes.Buffer) - var n int - switch test.encoding { - case "UTF-8": - n, err = p.WriteComment(buf, "# ", UTF8) - case "ISO-8859-1": - n, err = p.WriteComment(buf, "# ", ISO_8859_1) - } - c.Assert(err, IsNil) - s := string(buf.Bytes()) - c.Assert(n, Equals, len(test.output), Commentf("input=%q expected=%q obtained=%q", test.input, test.output, s)) - c.Assert(s, Equals, test.output, Commentf("input=%q expected=%q obtained=%q", test.input, test.output, s)) - } -} - -func (s *TestSuite) TestCustomExpansionExpression(c *C) { - testKeyValuePrePostfix(c, "*[", "]*", "key=value\nkey2=*[key]*", "key", "value", "key2", "value") -} - -func (s *TestSuite) TestPanicOn32BitIntOverflow(c *C) { - is32Bit = true - var min, max int64 = math.MinInt32 - 1, math.MaxInt32 + 1 - input := fmt.Sprintf("min=%d\nmax=%d", min, max) - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetInt64("min"), Equals, min) - c.Assert(p.MustGetInt64("max"), Equals, max) - c.Assert(func() { p.MustGetInt("min") }, PanicMatches, ".* out of range") - c.Assert(func() { p.MustGetInt("max") }, PanicMatches, ".* out of range") -} - -func (s *TestSuite) TestPanicOn32BitUintOverflow(c *C) { - is32Bit = true - var max uint64 = math.MaxUint32 + 1 - input := fmt.Sprintf("max=%d", max) - p, err := parse(input) - c.Assert(err, IsNil) - c.Assert(p.MustGetUint64("max"), Equals, max) - c.Assert(func() { p.MustGetUint("max") }, PanicMatches, ".* out of range") -} - -func (s *TestSuite) TestDeleteKey(c *C) { - input := "#comments should also be gone\nkey=to-be-deleted\nsecond=key" - p, err := parse(input) - c.Assert(err, IsNil) - c.Check(len(p.m), Equals, 2) - c.Check(len(p.c), Equals, 1) - c.Check(len(p.k), Equals, 2) - p.Delete("key") - c.Check(len(p.m), Equals, 1) - c.Check(len(p.c), Equals, 0) - c.Check(len(p.k), Equals, 1) -} - -func (s *TestSuite) TestDeleteUnknownKey(c *C) { - input := "#comments should also be gone\nkey=to-be-deleted" - p, err := parse(input) - c.Assert(err, IsNil) - c.Check(len(p.m), Equals, 1) - c.Check(len(p.c), Equals, 1) - c.Check(len(p.k), Equals, 1) - p.Delete("wrong-key") - c.Check(len(p.m), Equals, 1) - c.Check(len(p.c), Equals, 1) - c.Check(len(p.k), Equals, 1) -} - -// ---------------------------------------------------------------------------- - -// tests all combinations of delimiters, leading and/or trailing whitespace and newlines. -func testWhitespaceAndDelimiterCombinations(c *C, key, value string) { - whitespace := []string{"", " ", "\f", "\t"} - delimiters := []string{"", " ", "=", ":"} - newlines := []string{"", "\r", "\n", "\r\n"} - for _, dl := range delimiters { - for _, ws1 := range whitespace { - for _, ws2 := range whitespace { - for _, nl := range newlines { - // skip the one case where there is nothing between a key and a value - if ws1 == "" && dl == "" && ws2 == "" && value != "" { - continue - } - - input := fmt.Sprintf("%s%s%s%s%s%s", key, ws1, dl, ws2, value, nl) - testKeyValue(c, input, key, value) - } - } - } - } -} - -// tests whether key/value pairs exist for a given input. -// keyvalues is expected to be an even number of strings of "key", "value", ... -func testKeyValue(c *C, input string, keyvalues ...string) { - testKeyValuePrePostfix(c, "${", "}", input, keyvalues...) -} - -// tests whether key/value pairs exist for a given input. -// keyvalues is expected to be an even number of strings of "key", "value", ... -func testKeyValuePrePostfix(c *C, prefix, postfix, input string, keyvalues ...string) { - printf("%q\n", input) - - p, err := Load([]byte(input), ISO_8859_1) - c.Assert(err, IsNil) - p.Prefix = prefix - p.Postfix = postfix - assertKeyValues(c, input, p, keyvalues...) -} - -// tests whether key/value pairs exist for a given input. -// keyvalues is expected to be an even number of strings of "key", "value", ... -func assertKeyValues(c *C, input string, p *Properties, keyvalues ...string) { - c.Assert(p, NotNil) - c.Assert(2*p.Len(), Equals, len(keyvalues), Commentf("Odd number of key/value pairs.")) - - for i := 0; i < len(keyvalues); i += 2 { - key, value := keyvalues[i], keyvalues[i+1] - v, ok := p.Get(key) - c.Assert(ok, Equals, true, Commentf("No key %q found (input=%q)", key, input)) - c.Assert(v, Equals, value, Commentf("Value %q does not match %q (input=%q)", v, value, input)) - } -} - -// prints to stderr if the -verbose flag was given. -func printf(format string, args ...interface{}) { - if *verbose { - fmt.Fprintf(os.Stderr, format, args...) - } -} diff --git a/vendor/github.com/magiconair/properties/rangecheck.go b/vendor/github.com/magiconair/properties/rangecheck.go index 6df4e6ca..2e907d54 100644 --- a/vendor/github.com/magiconair/properties/rangecheck.go +++ b/vendor/github.com/magiconair/properties/rangecheck.go @@ -1,4 +1,4 @@ -// Copyright 2013-2014 Frank Schroeder. All rights reserved. +// Copyright 2017 Frank Schroeder. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/vendor/github.com/kr/pretty/License b/vendor/github.com/matryer/resync/LICENSE similarity index 89% rename from vendor/github.com/kr/pretty/License rename to vendor/github.com/matryer/resync/LICENSE index 05c783cc..e516db8f 100644 --- a/vendor/github.com/kr/pretty/License +++ b/vendor/github.com/matryer/resync/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright 2012 Keith Rarick +Copyright (c) 2016 Mat Ryer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/matryer/resync/README.md b/vendor/github.com/matryer/resync/README.md index 581b3ba9..9a1a83a5 100644 --- a/vendor/github.com/matryer/resync/README.md +++ b/vendor/github.com/matryer/resync/README.md @@ -4,11 +4,13 @@ * See [sync.Once](http://golang.org/pkg/sync/#Once) +Rather than adding this project as a dependency, consider [dropping](https://github.com/matryer/drop) this file into your project. + ## Example The following example examines how `resync.Once` could be used in a HTTP server situation. -``` +```go // use it just like sync.Once var once resync.Once diff --git a/vendor/github.com/matryer/resync/once_test.go b/vendor/github.com/matryer/resync/once_test.go deleted file mode 100644 index fa68a792..00000000 --- a/vendor/github.com/matryer/resync/once_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package resync_test - -import ( - "testing" - - "github.com/cheekybits/is" - "github.com/matryer/resync" -) - -func TestOnceReset(t *testing.T) { - is := is.New(t) - var calls int - var c resync.Once - c.Do(func() { - calls++ - }) - c.Do(func() { - calls++ - }) - c.Do(func() { - calls++ - }) - is.Equal(calls, 1) - c.Reset() - c.Do(func() { - calls++ - }) - c.Do(func() { - calls++ - }) - c.Do(func() { - calls++ - }) - is.Equal(calls, 2) -} diff --git a/vendor/github.com/miekg/dns/AUTHORS b/vendor/github.com/miekg/dns/AUTHORS new file mode 100644 index 00000000..19656835 --- /dev/null +++ b/vendor/github.com/miekg/dns/AUTHORS @@ -0,0 +1 @@ +Miek Gieben diff --git a/vendor/github.com/miekg/dns/CONTRIBUTORS b/vendor/github.com/miekg/dns/CONTRIBUTORS new file mode 100644 index 00000000..f77e8a89 --- /dev/null +++ b/vendor/github.com/miekg/dns/CONTRIBUTORS @@ -0,0 +1,9 @@ +Alex A. Skinner +Andrew Tunnell-Jones +Ask Bjørn Hansen +Dave Cheney +Dusty Wilson +Marek Majkowski +Peter van Dijk +Omri Bahumi +Alex Sergeyev diff --git a/vendor/github.com/miekg/dns/COPYRIGHT b/vendor/github.com/miekg/dns/COPYRIGHT new file mode 100644 index 00000000..35702b10 --- /dev/null +++ b/vendor/github.com/miekg/dns/COPYRIGHT @@ -0,0 +1,9 @@ +Copyright 2009 The Go Authors. All rights reserved. Use of this source code +is governed by a BSD-style license that can be found in the LICENSE file. +Extensions of the original work are copyright (c) 2011 Miek Gieben + +Copyright 2011 Miek Gieben. All rights reserved. Use of this source code is +governed by a BSD-style license that can be found in the LICENSE file. + +Copyright 2014 CloudFlare. All rights reserved. Use of this source code is +governed by a BSD-style license that can be found in the LICENSE file. diff --git a/vendor/github.com/miekg/dns/LICENSE b/vendor/github.com/miekg/dns/LICENSE new file mode 100644 index 00000000..5763fa7f --- /dev/null +++ b/vendor/github.com/miekg/dns/LICENSE @@ -0,0 +1,32 @@ +Extensions of the original work are copyright (c) 2011 Miek Gieben + +As this is fork of the official Go code the same license applies: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/miekg/dns/README.md b/vendor/github.com/miekg/dns/README.md new file mode 100644 index 00000000..cd5c1948 --- /dev/null +++ b/vendor/github.com/miekg/dns/README.md @@ -0,0 +1,160 @@ +[![Build Status](https://travis-ci.org/miekg/dns.svg?branch=master)](https://travis-ci.org/miekg/dns) +[![](https://godoc.org/github.com/miekg/dns?status.svg)](https://godoc.org/github.com/miekg/dns) + +# Alternative (more granular) approach to a DNS library + +> Less is more. + +Complete and usable DNS library. All widely used Resource Records are +supported, including the DNSSEC types. It follows a lean and mean philosophy. +If there is stuff you should know as a DNS programmer there isn't a convenience +function for it. Server side and client side programming is supported, i.e. you +can build servers and resolvers with it. + +We try to keep the "master" branch as sane as possible and at the bleeding edge +of standards, avoiding breaking changes wherever reasonable. We support the last +two versions of Go, currently: 1.6 and 1.7. + +# Goals + +* KISS; +* Fast; +* Small API. If it's easy to code in Go, don't make a function for it. + +# Users + +A not-so-up-to-date-list-that-may-be-actually-current: + +* https://github.com/coredns/coredns +* https://cloudflare.com +* https://github.com/abh/geodns +* http://www.statdns.com/ +* http://www.dnsinspect.com/ +* https://github.com/chuangbo/jianbing-dictionary-dns +* http://www.dns-lg.com/ +* https://github.com/fcambus/rrda +* https://github.com/kenshinx/godns +* https://github.com/skynetservices/skydns +* https://github.com/hashicorp/consul +* https://github.com/DevelopersPL/godnsagent +* https://github.com/duedil-ltd/discodns +* https://github.com/StalkR/dns-reverse-proxy +* https://github.com/tianon/rawdns +* https://mesosphere.github.io/mesos-dns/ +* https://pulse.turbobytes.com/ +* https://play.google.com/store/apps/details?id=com.turbobytes.dig +* https://github.com/fcambus/statzone +* https://github.com/benschw/dns-clb-go +* https://github.com/corny/dnscheck for http://public-dns.info/ +* https://namesmith.io +* https://github.com/miekg/unbound +* https://github.com/miekg/exdns +* https://dnslookup.org +* https://github.com/looterz/grimd +* https://github.com/phamhongviet/serf-dns +* https://github.com/mehrdadrad/mylg +* https://github.com/bamarni/dockness +* https://github.com/fffaraz/microdns +* http://quilt.io +* https://github.com/ipdcode/hades (JD.COM) +* https://github.com/StackExchange/dnscontrol/ + +Send pull request if you want to be listed here. + +# Features + +* UDP/TCP queries, IPv4 and IPv6; +* RFC 1035 zone file parsing ($INCLUDE, $ORIGIN, $TTL and $GENERATE (for all record types) are supported; +* Fast: + * Reply speed around ~ 80K qps (faster hardware results in more qps); + * Parsing RRs ~ 100K RR/s, that's 5M records in about 50 seconds; +* Server side programming (mimicking the net/http package); +* Client side programming; +* DNSSEC: signing, validating and key generation for DSA, RSA and ECDSA; +* EDNS0, NSID, Cookies; +* AXFR/IXFR; +* TSIG, SIG(0); +* DNS over TLS: optional encrypted connection between client and server; +* DNS name compression; +* Depends only on the standard library. + +Have fun! + +Miek Gieben - 2010-2012 - + +# Building + +Building is done with the `go` tool. If you have setup your GOPATH +correctly, the following should work: + + go get github.com/miekg/dns + go build github.com/miekg/dns + +## Examples + +A short "how to use the API" is at the beginning of doc.go (this also will show +when you call `godoc github.com/miekg/dns`). + +Example programs can be found in the `github.com/miekg/exdns` repository. + +## Supported RFCs + +*all of them* + +* 103{4,5} - DNS standard +* 1348 - NSAP record (removed the record) +* 1982 - Serial Arithmetic +* 1876 - LOC record +* 1995 - IXFR +* 1996 - DNS notify +* 2136 - DNS Update (dynamic updates) +* 2181 - RRset definition - there is no RRset type though, just []RR +* 2537 - RSAMD5 DNS keys +* 2065 - DNSSEC (updated in later RFCs) +* 2671 - EDNS record +* 2782 - SRV record +* 2845 - TSIG record +* 2915 - NAPTR record +* 2929 - DNS IANA Considerations +* 3110 - RSASHA1 DNS keys +* 3225 - DO bit (DNSSEC OK) +* 340{1,2,3} - NAPTR record +* 3445 - Limiting the scope of (DNS)KEY +* 3597 - Unknown RRs +* 403{3,4,5} - DNSSEC + validation functions +* 4255 - SSHFP record +* 4343 - Case insensitivity +* 4408 - SPF record +* 4509 - SHA256 Hash in DS +* 4592 - Wildcards in the DNS +* 4635 - HMAC SHA TSIG +* 4701 - DHCID +* 4892 - id.server +* 5001 - NSID +* 5155 - NSEC3 record +* 5205 - HIP record +* 5702 - SHA2 in the DNS +* 5936 - AXFR +* 5966 - TCP implementation recommendations +* 6605 - ECDSA +* 6725 - IANA Registry Update +* 6742 - ILNP DNS +* 6840 - Clarifications and Implementation Notes for DNS Security +* 6844 - CAA record +* 6891 - EDNS0 update +* 6895 - DNS IANA considerations +* 6975 - Algorithm Understanding in DNSSEC +* 7043 - EUI48/EUI64 records +* 7314 - DNS (EDNS) EXPIRE Option +* 7828 - edns-tcp-keepalive EDNS0 Option +* 7553 - URI record +* 7858 - DNS over TLS: Initiation and Performance Considerations (draft) +* 7873 - Domain Name System (DNS) Cookies (draft-ietf-dnsop-cookies) +* xxxx - EDNS0 DNS Update Lease (draft) + +## Loosely based upon + +* `ldns` +* `NSD` +* `Net::DNS` +* `GRONG` diff --git a/vendor/github.com/miekg/dns/client.go b/vendor/github.com/miekg/dns/client.go new file mode 100644 index 00000000..301dab9c --- /dev/null +++ b/vendor/github.com/miekg/dns/client.go @@ -0,0 +1,467 @@ +package dns + +// A client implementation. + +import ( + "bytes" + "crypto/tls" + "encoding/binary" + "io" + "net" + "time" +) + +const dnsTimeout time.Duration = 2 * time.Second +const tcpIdleTimeout time.Duration = 8 * time.Second + +// A Conn represents a connection to a DNS server. +type Conn struct { + net.Conn // a net.Conn holding the connection + UDPSize uint16 // minimum receive buffer for UDP messages + TsigSecret map[string]string // secret(s) for Tsig map[], zonename must be fully qualified + rtt time.Duration + t time.Time + tsigRequestMAC string +} + +// A Client defines parameters for a DNS client. +type Client struct { + Net string // if "tcp" or "tcp-tls" (DNS over TLS) a TCP query will be initiated, otherwise an UDP one (default is "" for UDP) + UDPSize uint16 // minimum receive buffer for UDP messages + TLSConfig *tls.Config // TLS connection configuration + Timeout time.Duration // a cumulative timeout for dial, write and read, defaults to 0 (disabled) - overrides DialTimeout, ReadTimeout and WriteTimeout when non-zero + DialTimeout time.Duration // net.DialTimeout, defaults to 2 seconds - overridden by Timeout when that value is non-zero + ReadTimeout time.Duration // net.Conn.SetReadTimeout value for connections, defaults to 2 seconds - overridden by Timeout when that value is non-zero + WriteTimeout time.Duration // net.Conn.SetWriteTimeout value for connections, defaults to 2 seconds - overridden by Timeout when that value is non-zero + TsigSecret map[string]string // secret(s) for Tsig map[], zonename must be fully qualified + SingleInflight bool // if true suppress multiple outstanding queries for the same Qname, Qtype and Qclass + group singleflight +} + +// Exchange performs a synchronous UDP query. It sends the message m to the address +// contained in a and waits for a reply. Exchange does not retry a failed query, nor +// will it fall back to TCP in case of truncation. +// See client.Exchange for more information on setting larger buffer sizes. +func Exchange(m *Msg, a string) (r *Msg, err error) { + var co *Conn + co, err = DialTimeout("udp", a, dnsTimeout) + if err != nil { + return nil, err + } + + defer co.Close() + + opt := m.IsEdns0() + // If EDNS0 is used use that for size. + if opt != nil && opt.UDPSize() >= MinMsgSize { + co.UDPSize = opt.UDPSize() + } + + co.SetWriteDeadline(time.Now().Add(dnsTimeout)) + if err = co.WriteMsg(m); err != nil { + return nil, err + } + + co.SetReadDeadline(time.Now().Add(dnsTimeout)) + r, err = co.ReadMsg() + if err == nil && r.Id != m.Id { + err = ErrId + } + return r, err +} + +// ExchangeConn performs a synchronous query. It sends the message m via the connection +// c and waits for a reply. The connection c is not closed by ExchangeConn. +// This function is going away, but can easily be mimicked: +// +// co := &dns.Conn{Conn: c} // c is your net.Conn +// co.WriteMsg(m) +// in, _ := co.ReadMsg() +// co.Close() +// +func ExchangeConn(c net.Conn, m *Msg) (r *Msg, err error) { + println("dns: this function is deprecated") + co := new(Conn) + co.Conn = c + if err = co.WriteMsg(m); err != nil { + return nil, err + } + r, err = co.ReadMsg() + if err == nil && r.Id != m.Id { + err = ErrId + } + return r, err +} + +// Exchange performs a synchronous query. It sends the message m to the address +// contained in a and waits for a reply. Basic use pattern with a *dns.Client: +// +// c := new(dns.Client) +// in, rtt, err := c.Exchange(message, "127.0.0.1:53") +// +// Exchange does not retry a failed query, nor will it fall back to TCP in +// case of truncation. +// It is up to the caller to create a message that allows for larger responses to be +// returned. Specifically this means adding an EDNS0 OPT RR that will advertise a larger +// buffer, see SetEdns0. Messages without an OPT RR will fallback to the historic limit +// of 512 bytes. +func (c *Client) Exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err error) { + if !c.SingleInflight { + return c.exchange(m, a) + } + // This adds a bunch of garbage, TODO(miek). + t := "nop" + if t1, ok := TypeToString[m.Question[0].Qtype]; ok { + t = t1 + } + cl := "nop" + if cl1, ok := ClassToString[m.Question[0].Qclass]; ok { + cl = cl1 + } + r, rtt, err, shared := c.group.Do(m.Question[0].Name+t+cl, func() (*Msg, time.Duration, error) { + return c.exchange(m, a) + }) + if r != nil && shared { + r = r.Copy() + } + if err != nil { + return r, rtt, err + } + return r, rtt, nil +} + +func (c *Client) dialTimeout() time.Duration { + if c.Timeout != 0 { + return c.Timeout + } + if c.DialTimeout != 0 { + return c.DialTimeout + } + return dnsTimeout +} + +func (c *Client) readTimeout() time.Duration { + if c.ReadTimeout != 0 { + return c.ReadTimeout + } + return dnsTimeout +} + +func (c *Client) writeTimeout() time.Duration { + if c.WriteTimeout != 0 { + return c.WriteTimeout + } + return dnsTimeout +} + +func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err error) { + var co *Conn + network := "udp" + tls := false + + switch c.Net { + case "tcp-tls": + network = "tcp" + tls = true + case "tcp4-tls": + network = "tcp4" + tls = true + case "tcp6-tls": + network = "tcp6" + tls = true + default: + if c.Net != "" { + network = c.Net + } + } + + var deadline time.Time + if c.Timeout != 0 { + deadline = time.Now().Add(c.Timeout) + } + + if tls { + co, err = DialTimeoutWithTLS(network, a, c.TLSConfig, c.dialTimeout()) + } else { + co, err = DialTimeout(network, a, c.dialTimeout()) + } + + if err != nil { + return nil, 0, err + } + defer co.Close() + + opt := m.IsEdns0() + // If EDNS0 is used use that for size. + if opt != nil && opt.UDPSize() >= MinMsgSize { + co.UDPSize = opt.UDPSize() + } + // Otherwise use the client's configured UDP size. + if opt == nil && c.UDPSize >= MinMsgSize { + co.UDPSize = c.UDPSize + } + + co.TsigSecret = c.TsigSecret + co.SetWriteDeadline(deadlineOrTimeout(deadline, c.writeTimeout())) + if err = co.WriteMsg(m); err != nil { + return nil, 0, err + } + + co.SetReadDeadline(deadlineOrTimeout(deadline, c.readTimeout())) + r, err = co.ReadMsg() + if err == nil && r.Id != m.Id { + err = ErrId + } + return r, co.rtt, err +} + +// ReadMsg reads a message from the connection co. +// If the received message contains a TSIG record the transaction +// signature is verified. +func (co *Conn) ReadMsg() (*Msg, error) { + p, err := co.ReadMsgHeader(nil) + if err != nil { + return nil, err + } + + m := new(Msg) + if err := m.Unpack(p); err != nil { + // If ErrTruncated was returned, we still want to allow the user to use + // the message, but naively they can just check err if they don't want + // to use a truncated message + if err == ErrTruncated { + return m, err + } + return nil, err + } + if t := m.IsTsig(); t != nil { + if _, ok := co.TsigSecret[t.Hdr.Name]; !ok { + return m, ErrSecret + } + // Need to work on the original message p, as that was used to calculate the tsig. + err = TsigVerify(p, co.TsigSecret[t.Hdr.Name], co.tsigRequestMAC, false) + } + return m, err +} + +// ReadMsgHeader reads a DNS message, parses and populates hdr (when hdr is not nil). +// Returns message as a byte slice to be parsed with Msg.Unpack later on. +// Note that error handling on the message body is not possible as only the header is parsed. +func (co *Conn) ReadMsgHeader(hdr *Header) ([]byte, error) { + var ( + p []byte + n int + err error + ) + + switch t := co.Conn.(type) { + case *net.TCPConn, *tls.Conn: + r := t.(io.Reader) + + // First two bytes specify the length of the entire message. + l, err := tcpMsgLen(r) + if err != nil { + return nil, err + } + p = make([]byte, l) + n, err = tcpRead(r, p) + co.rtt = time.Since(co.t) + default: + if co.UDPSize > MinMsgSize { + p = make([]byte, co.UDPSize) + } else { + p = make([]byte, MinMsgSize) + } + n, err = co.Read(p) + co.rtt = time.Since(co.t) + } + + if err != nil { + return nil, err + } else if n < headerSize { + return nil, ErrShortRead + } + + p = p[:n] + if hdr != nil { + dh, _, err := unpackMsgHdr(p, 0) + if err != nil { + return nil, err + } + *hdr = dh + } + return p, err +} + +// tcpMsgLen is a helper func to read first two bytes of stream as uint16 packet length. +func tcpMsgLen(t io.Reader) (int, error) { + p := []byte{0, 0} + n, err := t.Read(p) + if err != nil { + return 0, err + } + + // As seen with my local router/switch, retursn 1 byte on the above read, + // resulting a a ShortRead. Just write it out (instead of loop) and read the + // other byte. + if n == 1 { + n1, err := t.Read(p[1:]) + if err != nil { + return 0, err + } + n += n1 + } + + if n != 2 { + return 0, ErrShortRead + } + l := binary.BigEndian.Uint16(p) + if l == 0 { + return 0, ErrShortRead + } + return int(l), nil +} + +// tcpRead calls TCPConn.Read enough times to fill allocated buffer. +func tcpRead(t io.Reader, p []byte) (int, error) { + n, err := t.Read(p) + if err != nil { + return n, err + } + for n < len(p) { + j, err := t.Read(p[n:]) + if err != nil { + return n, err + } + n += j + } + return n, err +} + +// Read implements the net.Conn read method. +func (co *Conn) Read(p []byte) (n int, err error) { + if co.Conn == nil { + return 0, ErrConnEmpty + } + if len(p) < 2 { + return 0, io.ErrShortBuffer + } + switch t := co.Conn.(type) { + case *net.TCPConn, *tls.Conn: + r := t.(io.Reader) + + l, err := tcpMsgLen(r) + if err != nil { + return 0, err + } + if l > len(p) { + return int(l), io.ErrShortBuffer + } + return tcpRead(r, p[:l]) + } + // UDP connection + n, err = co.Conn.Read(p) + if err != nil { + return n, err + } + return n, err +} + +// WriteMsg sends a message through the connection co. +// If the message m contains a TSIG record the transaction +// signature is calculated. +func (co *Conn) WriteMsg(m *Msg) (err error) { + var out []byte + if t := m.IsTsig(); t != nil { + mac := "" + if _, ok := co.TsigSecret[t.Hdr.Name]; !ok { + return ErrSecret + } + out, mac, err = TsigGenerate(m, co.TsigSecret[t.Hdr.Name], co.tsigRequestMAC, false) + // Set for the next read, although only used in zone transfers + co.tsigRequestMAC = mac + } else { + out, err = m.Pack() + } + if err != nil { + return err + } + co.t = time.Now() + if _, err = co.Write(out); err != nil { + return err + } + return nil +} + +// Write implements the net.Conn Write method. +func (co *Conn) Write(p []byte) (n int, err error) { + switch t := co.Conn.(type) { + case *net.TCPConn, *tls.Conn: + w := t.(io.Writer) + + lp := len(p) + if lp < 2 { + return 0, io.ErrShortBuffer + } + if lp > MaxMsgSize { + return 0, &Error{err: "message too large"} + } + l := make([]byte, 2, lp+2) + binary.BigEndian.PutUint16(l, uint16(lp)) + p = append(l, p...) + n, err := io.Copy(w, bytes.NewReader(p)) + return int(n), err + } + n, err = co.Conn.Write(p) + return n, err +} + +// Dial connects to the address on the named network. +func Dial(network, address string) (conn *Conn, err error) { + conn = new(Conn) + conn.Conn, err = net.Dial(network, address) + if err != nil { + return nil, err + } + return conn, nil +} + +// DialTimeout acts like Dial but takes a timeout. +func DialTimeout(network, address string, timeout time.Duration) (conn *Conn, err error) { + conn = new(Conn) + conn.Conn, err = net.DialTimeout(network, address, timeout) + if err != nil { + return nil, err + } + return conn, nil +} + +// DialWithTLS connects to the address on the named network with TLS. +func DialWithTLS(network, address string, tlsConfig *tls.Config) (conn *Conn, err error) { + conn = new(Conn) + conn.Conn, err = tls.Dial(network, address, tlsConfig) + if err != nil { + return nil, err + } + return conn, nil +} + +// DialTimeoutWithTLS acts like DialWithTLS but takes a timeout. +func DialTimeoutWithTLS(network, address string, tlsConfig *tls.Config, timeout time.Duration) (conn *Conn, err error) { + var dialer net.Dialer + dialer.Timeout = timeout + + conn = new(Conn) + conn.Conn, err = tls.DialWithDialer(&dialer, network, address, tlsConfig) + if err != nil { + return nil, err + } + return conn, nil +} + +func deadlineOrTimeout(deadline time.Time, timeout time.Duration) time.Time { + if deadline.IsZero() { + return time.Now().Add(timeout) + } + return deadline +} diff --git a/vendor/github.com/miekg/dns/clientconfig.go b/vendor/github.com/miekg/dns/clientconfig.go new file mode 100644 index 00000000..0a1f5a92 --- /dev/null +++ b/vendor/github.com/miekg/dns/clientconfig.go @@ -0,0 +1,131 @@ +package dns + +import ( + "bufio" + "os" + "strconv" + "strings" +) + +// ClientConfig wraps the contents of the /etc/resolv.conf file. +type ClientConfig struct { + Servers []string // servers to use + Search []string // suffixes to append to local name + Port string // what port to use + Ndots int // number of dots in name to trigger absolute lookup + Timeout int // seconds before giving up on packet + Attempts int // lost packets before giving up on server, not used in the package dns +} + +// ClientConfigFromFile parses a resolv.conf(5) like file and returns +// a *ClientConfig. +func ClientConfigFromFile(resolvconf string) (*ClientConfig, error) { + file, err := os.Open(resolvconf) + if err != nil { + return nil, err + } + defer file.Close() + c := new(ClientConfig) + scanner := bufio.NewScanner(file) + c.Servers = make([]string, 0) + c.Search = make([]string, 0) + c.Port = "53" + c.Ndots = 1 + c.Timeout = 5 + c.Attempts = 2 + + for scanner.Scan() { + if err := scanner.Err(); err != nil { + return nil, err + } + line := scanner.Text() + f := strings.Fields(line) + if len(f) < 1 { + continue + } + switch f[0] { + case "nameserver": // add one name server + if len(f) > 1 { + // One more check: make sure server name is + // just an IP address. Otherwise we need DNS + // to look it up. + name := f[1] + c.Servers = append(c.Servers, name) + } + + case "domain": // set search path to just this domain + if len(f) > 1 { + c.Search = make([]string, 1) + c.Search[0] = f[1] + } else { + c.Search = make([]string, 0) + } + + case "search": // set search path to given servers + c.Search = make([]string, len(f)-1) + for i := 0; i < len(c.Search); i++ { + c.Search[i] = f[i+1] + } + + case "options": // magic options + for i := 1; i < len(f); i++ { + s := f[i] + switch { + case len(s) >= 6 && s[:6] == "ndots:": + n, _ := strconv.Atoi(s[6:]) + if n < 1 { + n = 1 + } + c.Ndots = n + case len(s) >= 8 && s[:8] == "timeout:": + n, _ := strconv.Atoi(s[8:]) + if n < 1 { + n = 1 + } + c.Timeout = n + case len(s) >= 8 && s[:9] == "attempts:": + n, _ := strconv.Atoi(s[9:]) + if n < 1 { + n = 1 + } + c.Attempts = n + case s == "rotate": + /* not imp */ + } + } + } + } + return c, nil +} + +// NameList returns all of the names that should be queried based on the +// config. It is based off of go's net/dns name building, but it does not +// check the length of the resulting names. +func (c *ClientConfig) NameList(name string) []string { + // if this domain is already fully qualified, no append needed. + if IsFqdn(name) { + return []string{name} + } + + // Check to see if the name has more labels than Ndots. Do this before making + // the domain fully qualified. + hasNdots := CountLabel(name) > c.Ndots + // Make the domain fully qualified. + name = Fqdn(name) + + // Make a list of names based off search. + names := []string{} + + // If name has enough dots, try that first. + if hasNdots { + names = append(names, name) + } + for _, s := range c.Search { + names = append(names, Fqdn(name+s)) + } + // If we didn't have enough dots, try after suffixes. + if !hasNdots { + names = append(names, name) + } + return names +} diff --git a/vendor/github.com/miekg/dns/compress_generate.go b/vendor/github.com/miekg/dns/compress_generate.go new file mode 100644 index 00000000..1a301e9f --- /dev/null +++ b/vendor/github.com/miekg/dns/compress_generate.go @@ -0,0 +1,184 @@ +//+build ignore + +// compression_generate.go is meant to run with go generate. It will use +// go/{importer,types} to track down all the RR struct types. Then for each type +// it will look to see if there are (compressible) names, if so it will add that +// type to compressionLenHelperType and comressionLenSearchType which "fake" the +// compression so that Len() is fast. +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/importer" + "go/types" + "log" + "os" +) + +var packageHdr = ` +// *** DO NOT MODIFY *** +// AUTOGENERATED BY go generate from compress_generate.go + +package dns + +` + +// getTypeStruct will take a type and the package scope, and return the +// (innermost) struct if the type is considered a RR type (currently defined as +// those structs beginning with a RR_Header, could be redefined as implementing +// the RR interface). The bool return value indicates if embedded structs were +// resolved. +func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { + st, ok := t.Underlying().(*types.Struct) + if !ok { + return nil, false + } + if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { + return st, false + } + if st.Field(0).Anonymous() { + st, _ := getTypeStruct(st.Field(0).Type(), scope) + return st, true + } + return nil, false +} + +func main() { + // Import and type-check the package + pkg, err := importer.Default().Import("github.com/miekg/dns") + fatalIfErr(err) + scope := pkg.Scope() + + domainTypes := map[string]bool{} // Types that have a domain name in them (either comressible or not). + cdomainTypes := map[string]bool{} // Types that have a compressible domain name in them (subset of domainType) + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + st, _ := getTypeStruct(o.Type(), scope) + if st == nil { + continue + } + if name == "PrivateRR" { + continue + } + + if scope.Lookup("Type"+o.Name()) == nil && o.Name() != "RFC3597" { + log.Fatalf("Constant Type%s does not exist.", o.Name()) + } + + for i := 1; i < st.NumFields(); i++ { + if _, ok := st.Field(i).Type().(*types.Slice); ok { + if st.Tag(i) == `dns:"domain-name"` { + domainTypes[o.Name()] = true + } + if st.Tag(i) == `dns:"cdomain-name"` { + cdomainTypes[o.Name()] = true + domainTypes[o.Name()] = true + } + continue + } + + switch { + case st.Tag(i) == `dns:"domain-name"`: + domainTypes[o.Name()] = true + case st.Tag(i) == `dns:"cdomain-name"`: + cdomainTypes[o.Name()] = true + domainTypes[o.Name()] = true + } + } + } + + b := &bytes.Buffer{} + b.WriteString(packageHdr) + + // compressionLenHelperType - all types that have domain-name/cdomain-name can be used for compressing names + + fmt.Fprint(b, "func compressionLenHelperType(c map[string]int, r RR) {\n") + fmt.Fprint(b, "switch x := r.(type) {\n") + for name, _ := range domainTypes { + o := scope.Lookup(name) + st, _ := getTypeStruct(o.Type(), scope) + + fmt.Fprintf(b, "case *%s:\n", name) + for i := 1; i < st.NumFields(); i++ { + out := func(s string) { fmt.Fprintf(b, "compressionLenHelper(c, x.%s)\n", st.Field(i).Name()) } + + if _, ok := st.Field(i).Type().(*types.Slice); ok { + switch st.Tag(i) { + case `dns:"domain-name"`: + fallthrough + case `dns:"cdomain-name"`: + // For HIP we need to slice over the elements in this slice. + fmt.Fprintf(b, `for i := range x.%s { + compressionLenHelper(c, x.%s[i]) + } +`, st.Field(i).Name(), st.Field(i).Name()) + } + continue + } + + switch { + case st.Tag(i) == `dns:"cdomain-name"`: + fallthrough + case st.Tag(i) == `dns:"domain-name"`: + out(st.Field(i).Name()) + } + } + } + fmt.Fprintln(b, "}\n}\n\n") + + // compressionLenSearchType - search cdomain-tags types for compressible names. + + fmt.Fprint(b, "func compressionLenSearchType(c map[string]int, r RR) (int, bool) {\n") + fmt.Fprint(b, "switch x := r.(type) {\n") + for name, _ := range cdomainTypes { + o := scope.Lookup(name) + st, _ := getTypeStruct(o.Type(), scope) + + fmt.Fprintf(b, "case *%s:\n", name) + j := 1 + for i := 1; i < st.NumFields(); i++ { + out := func(s string, j int) { + fmt.Fprintf(b, "k%d, ok%d := compressionLenSearch(c, x.%s)\n", j, j, st.Field(i).Name()) + } + + // There are no slice types with names that can be compressed. + + switch { + case st.Tag(i) == `dns:"cdomain-name"`: + out(st.Field(i).Name(), j) + j++ + } + } + k := "k1" + ok := "ok1" + for i := 2; i < j; i++ { + k += fmt.Sprintf(" + k%d", i) + ok += fmt.Sprintf(" && ok%d", i) + } + fmt.Fprintf(b, "return %s, %s\n", k, ok) + } + fmt.Fprintln(b, "}\nreturn 0, false\n}\n\n") + + // gofmt + res, err := format.Source(b.Bytes()) + if err != nil { + b.WriteTo(os.Stderr) + log.Fatal(err) + } + + f, err := os.Create("zcompress.go") + fatalIfErr(err) + defer f.Close() + f.Write(res) +} + +func fatalIfErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/vendor/github.com/miekg/dns/dane.go b/vendor/github.com/miekg/dns/dane.go new file mode 100644 index 00000000..8c4a14ef --- /dev/null +++ b/vendor/github.com/miekg/dns/dane.go @@ -0,0 +1,43 @@ +package dns + +import ( + "crypto/sha256" + "crypto/sha512" + "crypto/x509" + "encoding/hex" + "errors" +) + +// CertificateToDANE converts a certificate to a hex string as used in the TLSA or SMIMEA records. +func CertificateToDANE(selector, matchingType uint8, cert *x509.Certificate) (string, error) { + switch matchingType { + case 0: + switch selector { + case 0: + return hex.EncodeToString(cert.Raw), nil + case 1: + return hex.EncodeToString(cert.RawSubjectPublicKeyInfo), nil + } + case 1: + h := sha256.New() + switch selector { + case 0: + h.Write(cert.Raw) + return hex.EncodeToString(h.Sum(nil)), nil + case 1: + h.Write(cert.RawSubjectPublicKeyInfo) + return hex.EncodeToString(h.Sum(nil)), nil + } + case 2: + h := sha512.New() + switch selector { + case 0: + h.Write(cert.Raw) + return hex.EncodeToString(h.Sum(nil)), nil + case 1: + h.Write(cert.RawSubjectPublicKeyInfo) + return hex.EncodeToString(h.Sum(nil)), nil + } + } + return "", errors.New("dns: bad MatchingType or Selector") +} diff --git a/vendor/github.com/miekg/dns/defaults.go b/vendor/github.com/miekg/dns/defaults.go new file mode 100644 index 00000000..3308ec83 --- /dev/null +++ b/vendor/github.com/miekg/dns/defaults.go @@ -0,0 +1,282 @@ +package dns + +import ( + "errors" + "net" + "strconv" +) + +const hexDigit = "0123456789abcdef" + +// Everything is assumed in ClassINET. + +// SetReply creates a reply message from a request message. +func (dns *Msg) SetReply(request *Msg) *Msg { + dns.Id = request.Id + dns.RecursionDesired = request.RecursionDesired // Copy rd bit + dns.Response = true + dns.Opcode = OpcodeQuery + dns.Rcode = RcodeSuccess + if len(request.Question) > 0 { + dns.Question = make([]Question, 1) + dns.Question[0] = request.Question[0] + } + return dns +} + +// SetQuestion creates a question message, it sets the Question +// section, generates an Id and sets the RecursionDesired (RD) +// bit to true. +func (dns *Msg) SetQuestion(z string, t uint16) *Msg { + dns.Id = Id() + dns.RecursionDesired = true + dns.Question = make([]Question, 1) + dns.Question[0] = Question{z, t, ClassINET} + return dns +} + +// SetNotify creates a notify message, it sets the Question +// section, generates an Id and sets the Authoritative (AA) +// bit to true. +func (dns *Msg) SetNotify(z string) *Msg { + dns.Opcode = OpcodeNotify + dns.Authoritative = true + dns.Id = Id() + dns.Question = make([]Question, 1) + dns.Question[0] = Question{z, TypeSOA, ClassINET} + return dns +} + +// SetRcode creates an error message suitable for the request. +func (dns *Msg) SetRcode(request *Msg, rcode int) *Msg { + dns.SetReply(request) + dns.Rcode = rcode + return dns +} + +// SetRcodeFormatError creates a message with FormError set. +func (dns *Msg) SetRcodeFormatError(request *Msg) *Msg { + dns.Rcode = RcodeFormatError + dns.Opcode = OpcodeQuery + dns.Response = true + dns.Authoritative = false + dns.Id = request.Id + return dns +} + +// SetUpdate makes the message a dynamic update message. It +// sets the ZONE section to: z, TypeSOA, ClassINET. +func (dns *Msg) SetUpdate(z string) *Msg { + dns.Id = Id() + dns.Response = false + dns.Opcode = OpcodeUpdate + dns.Compress = false // BIND9 cannot handle compression + dns.Question = make([]Question, 1) + dns.Question[0] = Question{z, TypeSOA, ClassINET} + return dns +} + +// SetIxfr creates message for requesting an IXFR. +func (dns *Msg) SetIxfr(z string, serial uint32, ns, mbox string) *Msg { + dns.Id = Id() + dns.Question = make([]Question, 1) + dns.Ns = make([]RR, 1) + s := new(SOA) + s.Hdr = RR_Header{z, TypeSOA, ClassINET, defaultTtl, 0} + s.Serial = serial + s.Ns = ns + s.Mbox = mbox + dns.Question[0] = Question{z, TypeIXFR, ClassINET} + dns.Ns[0] = s + return dns +} + +// SetAxfr creates message for requesting an AXFR. +func (dns *Msg) SetAxfr(z string) *Msg { + dns.Id = Id() + dns.Question = make([]Question, 1) + dns.Question[0] = Question{z, TypeAXFR, ClassINET} + return dns +} + +// SetTsig appends a TSIG RR to the message. +// This is only a skeleton TSIG RR that is added as the last RR in the +// additional section. The Tsig is calculated when the message is being send. +func (dns *Msg) SetTsig(z, algo string, fudge uint16, timesigned int64) *Msg { + t := new(TSIG) + t.Hdr = RR_Header{z, TypeTSIG, ClassANY, 0, 0} + t.Algorithm = algo + t.Fudge = fudge + t.TimeSigned = uint64(timesigned) + t.OrigId = dns.Id + dns.Extra = append(dns.Extra, t) + return dns +} + +// SetEdns0 appends a EDNS0 OPT RR to the message. +// TSIG should always the last RR in a message. +func (dns *Msg) SetEdns0(udpsize uint16, do bool) *Msg { + e := new(OPT) + e.Hdr.Name = "." + e.Hdr.Rrtype = TypeOPT + e.SetUDPSize(udpsize) + if do { + e.SetDo() + } + dns.Extra = append(dns.Extra, e) + return dns +} + +// IsTsig checks if the message has a TSIG record as the last record +// in the additional section. It returns the TSIG record found or nil. +func (dns *Msg) IsTsig() *TSIG { + if len(dns.Extra) > 0 { + if dns.Extra[len(dns.Extra)-1].Header().Rrtype == TypeTSIG { + return dns.Extra[len(dns.Extra)-1].(*TSIG) + } + } + return nil +} + +// IsEdns0 checks if the message has a EDNS0 (OPT) record, any EDNS0 +// record in the additional section will do. It returns the OPT record +// found or nil. +func (dns *Msg) IsEdns0() *OPT { + // EDNS0 is at the end of the additional section, start there. + // We might want to change this to *only* look at the last two + // records. So we see TSIG and/or OPT - this a slightly bigger + // change though. + for i := len(dns.Extra) - 1; i >= 0; i-- { + if dns.Extra[i].Header().Rrtype == TypeOPT { + return dns.Extra[i].(*OPT) + } + } + return nil +} + +// IsDomainName checks if s is a valid domain name, it returns the number of +// labels and true, when a domain name is valid. Note that non fully qualified +// domain name is considered valid, in this case the last label is counted in +// the number of labels. When false is returned the number of labels is not +// defined. Also note that this function is extremely liberal; almost any +// string is a valid domain name as the DNS is 8 bit protocol. It checks if each +// label fits in 63 characters, but there is no length check for the entire +// string s. I.e. a domain name longer than 255 characters is considered valid. +func IsDomainName(s string) (labels int, ok bool) { + _, labels, err := packDomainName(s, nil, 0, nil, false) + return labels, err == nil +} + +// IsSubDomain checks if child is indeed a child of the parent. If child and parent +// are the same domain true is returned as well. +func IsSubDomain(parent, child string) bool { + // Entire child is contained in parent + return CompareDomainName(parent, child) == CountLabel(parent) +} + +// IsMsg sanity checks buf and returns an error if it isn't a valid DNS packet. +// The checking is performed on the binary payload. +func IsMsg(buf []byte) error { + // Header + if len(buf) < 12 { + return errors.New("dns: bad message header") + } + // Header: Opcode + // TODO(miek): more checks here, e.g. check all header bits. + return nil +} + +// IsFqdn checks if a domain name is fully qualified. +func IsFqdn(s string) bool { + l := len(s) + if l == 0 { + return false + } + return s[l-1] == '.' +} + +// IsRRset checks if a set of RRs is a valid RRset as defined by RFC 2181. +// This means the RRs need to have the same type, name, and class. Returns true +// if the RR set is valid, otherwise false. +func IsRRset(rrset []RR) bool { + if len(rrset) == 0 { + return false + } + if len(rrset) == 1 { + return true + } + rrHeader := rrset[0].Header() + rrType := rrHeader.Rrtype + rrClass := rrHeader.Class + rrName := rrHeader.Name + + for _, rr := range rrset[1:] { + curRRHeader := rr.Header() + if curRRHeader.Rrtype != rrType || curRRHeader.Class != rrClass || curRRHeader.Name != rrName { + // Mismatch between the records, so this is not a valid rrset for + //signing/verifying + return false + } + } + + return true +} + +// Fqdn return the fully qualified domain name from s. +// If s is already fully qualified, it behaves as the identity function. +func Fqdn(s string) string { + if IsFqdn(s) { + return s + } + return s + "." +} + +// Copied from the official Go code. + +// ReverseAddr returns the in-addr.arpa. or ip6.arpa. hostname of the IP +// address suitable for reverse DNS (PTR) record lookups or an error if it fails +// to parse the IP address. +func ReverseAddr(addr string) (arpa string, err error) { + ip := net.ParseIP(addr) + if ip == nil { + return "", &Error{err: "unrecognized address: " + addr} + } + if ip.To4() != nil { + return strconv.Itoa(int(ip[15])) + "." + strconv.Itoa(int(ip[14])) + "." + strconv.Itoa(int(ip[13])) + "." + + strconv.Itoa(int(ip[12])) + ".in-addr.arpa.", nil + } + // Must be IPv6 + buf := make([]byte, 0, len(ip)*4+len("ip6.arpa.")) + // Add it, in reverse, to the buffer + for i := len(ip) - 1; i >= 0; i-- { + v := ip[i] + buf = append(buf, hexDigit[v&0xF]) + buf = append(buf, '.') + buf = append(buf, hexDigit[v>>4]) + buf = append(buf, '.') + } + // Append "ip6.arpa." and return (buf already has the final .) + buf = append(buf, "ip6.arpa."...) + return string(buf), nil +} + +// String returns the string representation for the type t. +func (t Type) String() string { + if t1, ok := TypeToString[uint16(t)]; ok { + return t1 + } + return "TYPE" + strconv.Itoa(int(t)) +} + +// String returns the string representation for the class c. +func (c Class) String() string { + if c1, ok := ClassToString[uint16(c)]; ok { + return c1 + } + return "CLASS" + strconv.Itoa(int(c)) +} + +// String returns the string representation for the name n. +func (n Name) String() string { + return sprintName(string(n)) +} diff --git a/vendor/github.com/miekg/dns/dns.go b/vendor/github.com/miekg/dns/dns.go new file mode 100644 index 00000000..b3292287 --- /dev/null +++ b/vendor/github.com/miekg/dns/dns.go @@ -0,0 +1,104 @@ +package dns + +import "strconv" + +const ( + year68 = 1 << 31 // For RFC1982 (Serial Arithmetic) calculations in 32 bits. + defaultTtl = 3600 // Default internal TTL. + + DefaultMsgSize = 4096 // DefaultMsgSize is the standard default for messages larger than 512 bytes. + MinMsgSize = 512 // MinMsgSize is the minimal size of a DNS packet. + MaxMsgSize = 65535 // MaxMsgSize is the largest possible DNS packet. +) + +// Error represents a DNS error. +type Error struct{ err string } + +func (e *Error) Error() string { + if e == nil { + return "dns: " + } + return "dns: " + e.err +} + +// An RR represents a resource record. +type RR interface { + // Header returns the header of an resource record. The header contains + // everything up to the rdata. + Header() *RR_Header + // String returns the text representation of the resource record. + String() string + + // copy returns a copy of the RR + copy() RR + // len returns the length (in octets) of the uncompressed RR in wire format. + len() int + // pack packs an RR into wire format. + pack([]byte, int, map[string]int, bool) (int, error) +} + +// RR_Header is the header all DNS resource records share. +type RR_Header struct { + Name string `dns:"cdomain-name"` + Rrtype uint16 + Class uint16 + Ttl uint32 + Rdlength uint16 // Length of data after header. +} + +// Header returns itself. This is here to make RR_Header implements the RR interface. +func (h *RR_Header) Header() *RR_Header { return h } + +// Just to implement the RR interface. +func (h *RR_Header) copy() RR { return nil } + +func (h *RR_Header) copyHeader() *RR_Header { + r := new(RR_Header) + r.Name = h.Name + r.Rrtype = h.Rrtype + r.Class = h.Class + r.Ttl = h.Ttl + r.Rdlength = h.Rdlength + return r +} + +func (h *RR_Header) String() string { + var s string + + if h.Rrtype == TypeOPT { + s = ";" + // and maybe other things + } + + s += sprintName(h.Name) + "\t" + s += strconv.FormatInt(int64(h.Ttl), 10) + "\t" + s += Class(h.Class).String() + "\t" + s += Type(h.Rrtype).String() + "\t" + return s +} + +func (h *RR_Header) len() int { + l := len(h.Name) + 1 + l += 10 // rrtype(2) + class(2) + ttl(4) + rdlength(2) + return l +} + +// ToRFC3597 converts a known RR to the unknown RR representation from RFC 3597. +func (rr *RFC3597) ToRFC3597(r RR) error { + buf := make([]byte, r.len()*2) + off, err := PackRR(r, buf, 0, nil, false) + if err != nil { + return err + } + buf = buf[:off] + if int(r.Header().Rdlength) > off { + return ErrBuf + } + + rfc3597, _, err := unpackRFC3597(*r.Header(), buf, off-int(r.Header().Rdlength)) + if err != nil { + return err + } + *rr = *rfc3597.(*RFC3597) + return nil +} diff --git a/vendor/github.com/miekg/dns/dnssec.go b/vendor/github.com/miekg/dns/dnssec.go new file mode 100644 index 00000000..3bd55388 --- /dev/null +++ b/vendor/github.com/miekg/dns/dnssec.go @@ -0,0 +1,720 @@ +package dns + +import ( + "bytes" + "crypto" + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + _ "crypto/md5" + "crypto/rand" + "crypto/rsa" + _ "crypto/sha1" + _ "crypto/sha256" + _ "crypto/sha512" + "encoding/asn1" + "encoding/binary" + "encoding/hex" + "math/big" + "sort" + "strings" + "time" +) + +// DNSSEC encryption algorithm codes. +const ( + _ uint8 = iota + RSAMD5 + DH + DSA + _ // Skip 4, RFC 6725, section 2.1 + RSASHA1 + DSANSEC3SHA1 + RSASHA1NSEC3SHA1 + RSASHA256 + _ // Skip 9, RFC 6725, section 2.1 + RSASHA512 + _ // Skip 11, RFC 6725, section 2.1 + ECCGOST + ECDSAP256SHA256 + ECDSAP384SHA384 + INDIRECT uint8 = 252 + PRIVATEDNS uint8 = 253 // Private (experimental keys) + PRIVATEOID uint8 = 254 +) + +// AlgorithmToString is a map of algorithm IDs to algorithm names. +var AlgorithmToString = map[uint8]string{ + RSAMD5: "RSAMD5", + DH: "DH", + DSA: "DSA", + RSASHA1: "RSASHA1", + DSANSEC3SHA1: "DSA-NSEC3-SHA1", + RSASHA1NSEC3SHA1: "RSASHA1-NSEC3-SHA1", + RSASHA256: "RSASHA256", + RSASHA512: "RSASHA512", + ECCGOST: "ECC-GOST", + ECDSAP256SHA256: "ECDSAP256SHA256", + ECDSAP384SHA384: "ECDSAP384SHA384", + INDIRECT: "INDIRECT", + PRIVATEDNS: "PRIVATEDNS", + PRIVATEOID: "PRIVATEOID", +} + +// StringToAlgorithm is the reverse of AlgorithmToString. +var StringToAlgorithm = reverseInt8(AlgorithmToString) + +// AlgorithmToHash is a map of algorithm crypto hash IDs to crypto.Hash's. +var AlgorithmToHash = map[uint8]crypto.Hash{ + RSAMD5: crypto.MD5, // Deprecated in RFC 6725 + RSASHA1: crypto.SHA1, + RSASHA1NSEC3SHA1: crypto.SHA1, + RSASHA256: crypto.SHA256, + ECDSAP256SHA256: crypto.SHA256, + ECDSAP384SHA384: crypto.SHA384, + RSASHA512: crypto.SHA512, +} + +// DNSSEC hashing algorithm codes. +const ( + _ uint8 = iota + SHA1 // RFC 4034 + SHA256 // RFC 4509 + GOST94 // RFC 5933 + SHA384 // Experimental + SHA512 // Experimental +) + +// HashToString is a map of hash IDs to names. +var HashToString = map[uint8]string{ + SHA1: "SHA1", + SHA256: "SHA256", + GOST94: "GOST94", + SHA384: "SHA384", + SHA512: "SHA512", +} + +// StringToHash is a map of names to hash IDs. +var StringToHash = reverseInt8(HashToString) + +// DNSKEY flag values. +const ( + SEP = 1 + REVOKE = 1 << 7 + ZONE = 1 << 8 +) + +// The RRSIG needs to be converted to wireformat with some of the rdata (the signature) missing. +type rrsigWireFmt struct { + TypeCovered uint16 + Algorithm uint8 + Labels uint8 + OrigTtl uint32 + Expiration uint32 + Inception uint32 + KeyTag uint16 + SignerName string `dns:"domain-name"` + /* No Signature */ +} + +// Used for converting DNSKEY's rdata to wirefmt. +type dnskeyWireFmt struct { + Flags uint16 + Protocol uint8 + Algorithm uint8 + PublicKey string `dns:"base64"` + /* Nothing is left out */ +} + +func divRoundUp(a, b int) int { + return (a + b - 1) / b +} + +// KeyTag calculates the keytag (or key-id) of the DNSKEY. +func (k *DNSKEY) KeyTag() uint16 { + if k == nil { + return 0 + } + var keytag int + switch k.Algorithm { + case RSAMD5: + // Look at the bottom two bytes of the modules, which the last + // item in the pubkey. We could do this faster by looking directly + // at the base64 values. But I'm lazy. + modulus, _ := fromBase64([]byte(k.PublicKey)) + if len(modulus) > 1 { + x := binary.BigEndian.Uint16(modulus[len(modulus)-2:]) + keytag = int(x) + } + default: + keywire := new(dnskeyWireFmt) + keywire.Flags = k.Flags + keywire.Protocol = k.Protocol + keywire.Algorithm = k.Algorithm + keywire.PublicKey = k.PublicKey + wire := make([]byte, DefaultMsgSize) + n, err := packKeyWire(keywire, wire) + if err != nil { + return 0 + } + wire = wire[:n] + for i, v := range wire { + if i&1 != 0 { + keytag += int(v) // must be larger than uint32 + } else { + keytag += int(v) << 8 + } + } + keytag += (keytag >> 16) & 0xFFFF + keytag &= 0xFFFF + } + return uint16(keytag) +} + +// ToDS converts a DNSKEY record to a DS record. +func (k *DNSKEY) ToDS(h uint8) *DS { + if k == nil { + return nil + } + ds := new(DS) + ds.Hdr.Name = k.Hdr.Name + ds.Hdr.Class = k.Hdr.Class + ds.Hdr.Rrtype = TypeDS + ds.Hdr.Ttl = k.Hdr.Ttl + ds.Algorithm = k.Algorithm + ds.DigestType = h + ds.KeyTag = k.KeyTag() + + keywire := new(dnskeyWireFmt) + keywire.Flags = k.Flags + keywire.Protocol = k.Protocol + keywire.Algorithm = k.Algorithm + keywire.PublicKey = k.PublicKey + wire := make([]byte, DefaultMsgSize) + n, err := packKeyWire(keywire, wire) + if err != nil { + return nil + } + wire = wire[:n] + + owner := make([]byte, 255) + off, err1 := PackDomainName(strings.ToLower(k.Hdr.Name), owner, 0, nil, false) + if err1 != nil { + return nil + } + owner = owner[:off] + // RFC4034: + // digest = digest_algorithm( DNSKEY owner name | DNSKEY RDATA); + // "|" denotes concatenation + // DNSKEY RDATA = Flags | Protocol | Algorithm | Public Key. + + var hash crypto.Hash + switch h { + case SHA1: + hash = crypto.SHA1 + case SHA256: + hash = crypto.SHA256 + case SHA384: + hash = crypto.SHA384 + case SHA512: + hash = crypto.SHA512 + default: + return nil + } + + s := hash.New() + s.Write(owner) + s.Write(wire) + ds.Digest = hex.EncodeToString(s.Sum(nil)) + return ds +} + +// ToCDNSKEY converts a DNSKEY record to a CDNSKEY record. +func (k *DNSKEY) ToCDNSKEY() *CDNSKEY { + c := &CDNSKEY{DNSKEY: *k} + c.Hdr = *k.Hdr.copyHeader() + c.Hdr.Rrtype = TypeCDNSKEY + return c +} + +// ToCDS converts a DS record to a CDS record. +func (d *DS) ToCDS() *CDS { + c := &CDS{DS: *d} + c.Hdr = *d.Hdr.copyHeader() + c.Hdr.Rrtype = TypeCDS + return c +} + +// Sign signs an RRSet. The signature needs to be filled in with the values: +// Inception, Expiration, KeyTag, SignerName and Algorithm. The rest is copied +// from the RRset. Sign returns a non-nill error when the signing went OK. +// There is no check if RRSet is a proper (RFC 2181) RRSet. If OrigTTL is non +// zero, it is used as-is, otherwise the TTL of the RRset is used as the +// OrigTTL. +func (rr *RRSIG) Sign(k crypto.Signer, rrset []RR) error { + if k == nil { + return ErrPrivKey + } + // s.Inception and s.Expiration may be 0 (rollover etc.), the rest must be set + if rr.KeyTag == 0 || len(rr.SignerName) == 0 || rr.Algorithm == 0 { + return ErrKey + } + + rr.Hdr.Rrtype = TypeRRSIG + rr.Hdr.Name = rrset[0].Header().Name + rr.Hdr.Class = rrset[0].Header().Class + if rr.OrigTtl == 0 { // If set don't override + rr.OrigTtl = rrset[0].Header().Ttl + } + rr.TypeCovered = rrset[0].Header().Rrtype + rr.Labels = uint8(CountLabel(rrset[0].Header().Name)) + + if strings.HasPrefix(rrset[0].Header().Name, "*") { + rr.Labels-- // wildcard, remove from label count + } + + sigwire := new(rrsigWireFmt) + sigwire.TypeCovered = rr.TypeCovered + sigwire.Algorithm = rr.Algorithm + sigwire.Labels = rr.Labels + sigwire.OrigTtl = rr.OrigTtl + sigwire.Expiration = rr.Expiration + sigwire.Inception = rr.Inception + sigwire.KeyTag = rr.KeyTag + // For signing, lowercase this name + sigwire.SignerName = strings.ToLower(rr.SignerName) + + // Create the desired binary blob + signdata := make([]byte, DefaultMsgSize) + n, err := packSigWire(sigwire, signdata) + if err != nil { + return err + } + signdata = signdata[:n] + wire, err := rawSignatureData(rrset, rr) + if err != nil { + return err + } + + hash, ok := AlgorithmToHash[rr.Algorithm] + if !ok { + return ErrAlg + } + + h := hash.New() + h.Write(signdata) + h.Write(wire) + + signature, err := sign(k, h.Sum(nil), hash, rr.Algorithm) + if err != nil { + return err + } + + rr.Signature = toBase64(signature) + + return nil +} + +func sign(k crypto.Signer, hashed []byte, hash crypto.Hash, alg uint8) ([]byte, error) { + signature, err := k.Sign(rand.Reader, hashed, hash) + if err != nil { + return nil, err + } + + switch alg { + case RSASHA1, RSASHA1NSEC3SHA1, RSASHA256, RSASHA512: + return signature, nil + + case ECDSAP256SHA256, ECDSAP384SHA384: + ecdsaSignature := &struct { + R, S *big.Int + }{} + if _, err := asn1.Unmarshal(signature, ecdsaSignature); err != nil { + return nil, err + } + + var intlen int + switch alg { + case ECDSAP256SHA256: + intlen = 32 + case ECDSAP384SHA384: + intlen = 48 + } + + signature := intToBytes(ecdsaSignature.R, intlen) + signature = append(signature, intToBytes(ecdsaSignature.S, intlen)...) + return signature, nil + + // There is no defined interface for what a DSA backed crypto.Signer returns + case DSA, DSANSEC3SHA1: + // t := divRoundUp(divRoundUp(p.PublicKey.Y.BitLen(), 8)-64, 8) + // signature := []byte{byte(t)} + // signature = append(signature, intToBytes(r1, 20)...) + // signature = append(signature, intToBytes(s1, 20)...) + // rr.Signature = signature + } + + return nil, ErrAlg +} + +// Verify validates an RRSet with the signature and key. This is only the +// cryptographic test, the signature validity period must be checked separately. +// This function copies the rdata of some RRs (to lowercase domain names) for the validation to work. +func (rr *RRSIG) Verify(k *DNSKEY, rrset []RR) error { + // First the easy checks + if !IsRRset(rrset) { + return ErrRRset + } + if rr.KeyTag != k.KeyTag() { + return ErrKey + } + if rr.Hdr.Class != k.Hdr.Class { + return ErrKey + } + if rr.Algorithm != k.Algorithm { + return ErrKey + } + if strings.ToLower(rr.SignerName) != strings.ToLower(k.Hdr.Name) { + return ErrKey + } + if k.Protocol != 3 { + return ErrKey + } + + // IsRRset checked that we have at least one RR and that the RRs in + // the set have consistent type, class, and name. Also check that type and + // class matches the RRSIG record. + if rrset[0].Header().Class != rr.Hdr.Class { + return ErrRRset + } + if rrset[0].Header().Rrtype != rr.TypeCovered { + return ErrRRset + } + + // RFC 4035 5.3.2. Reconstructing the Signed Data + // Copy the sig, except the rrsig data + sigwire := new(rrsigWireFmt) + sigwire.TypeCovered = rr.TypeCovered + sigwire.Algorithm = rr.Algorithm + sigwire.Labels = rr.Labels + sigwire.OrigTtl = rr.OrigTtl + sigwire.Expiration = rr.Expiration + sigwire.Inception = rr.Inception + sigwire.KeyTag = rr.KeyTag + sigwire.SignerName = strings.ToLower(rr.SignerName) + // Create the desired binary blob + signeddata := make([]byte, DefaultMsgSize) + n, err := packSigWire(sigwire, signeddata) + if err != nil { + return err + } + signeddata = signeddata[:n] + wire, err := rawSignatureData(rrset, rr) + if err != nil { + return err + } + + sigbuf := rr.sigBuf() // Get the binary signature data + if rr.Algorithm == PRIVATEDNS { // PRIVATEOID + // TODO(miek) + // remove the domain name and assume its ours? + } + + hash, ok := AlgorithmToHash[rr.Algorithm] + if !ok { + return ErrAlg + } + + switch rr.Algorithm { + case RSASHA1, RSASHA1NSEC3SHA1, RSASHA256, RSASHA512, RSAMD5: + // TODO(mg): this can be done quicker, ie. cache the pubkey data somewhere?? + pubkey := k.publicKeyRSA() // Get the key + if pubkey == nil { + return ErrKey + } + + h := hash.New() + h.Write(signeddata) + h.Write(wire) + return rsa.VerifyPKCS1v15(pubkey, hash, h.Sum(nil), sigbuf) + + case ECDSAP256SHA256, ECDSAP384SHA384: + pubkey := k.publicKeyECDSA() + if pubkey == nil { + return ErrKey + } + + // Split sigbuf into the r and s coordinates + r := new(big.Int).SetBytes(sigbuf[:len(sigbuf)/2]) + s := new(big.Int).SetBytes(sigbuf[len(sigbuf)/2:]) + + h := hash.New() + h.Write(signeddata) + h.Write(wire) + if ecdsa.Verify(pubkey, h.Sum(nil), r, s) { + return nil + } + return ErrSig + + default: + return ErrAlg + } +} + +// ValidityPeriod uses RFC1982 serial arithmetic to calculate +// if a signature period is valid. If t is the zero time, the +// current time is taken other t is. Returns true if the signature +// is valid at the given time, otherwise returns false. +func (rr *RRSIG) ValidityPeriod(t time.Time) bool { + var utc int64 + if t.IsZero() { + utc = time.Now().UTC().Unix() + } else { + utc = t.UTC().Unix() + } + modi := (int64(rr.Inception) - utc) / year68 + mode := (int64(rr.Expiration) - utc) / year68 + ti := int64(rr.Inception) + (modi * year68) + te := int64(rr.Expiration) + (mode * year68) + return ti <= utc && utc <= te +} + +// Return the signatures base64 encodedig sigdata as a byte slice. +func (rr *RRSIG) sigBuf() []byte { + sigbuf, err := fromBase64([]byte(rr.Signature)) + if err != nil { + return nil + } + return sigbuf +} + +// publicKeyRSA returns the RSA public key from a DNSKEY record. +func (k *DNSKEY) publicKeyRSA() *rsa.PublicKey { + keybuf, err := fromBase64([]byte(k.PublicKey)) + if err != nil { + return nil + } + + // RFC 2537/3110, section 2. RSA Public KEY Resource Records + // Length is in the 0th byte, unless its zero, then it + // it in bytes 1 and 2 and its a 16 bit number + explen := uint16(keybuf[0]) + keyoff := 1 + if explen == 0 { + explen = uint16(keybuf[1])<<8 | uint16(keybuf[2]) + keyoff = 3 + } + pubkey := new(rsa.PublicKey) + + pubkey.N = big.NewInt(0) + shift := uint64((explen - 1) * 8) + expo := uint64(0) + for i := int(explen - 1); i > 0; i-- { + expo += uint64(keybuf[keyoff+i]) << shift + shift -= 8 + } + // Remainder + expo += uint64(keybuf[keyoff]) + if expo > (2<<31)+1 { + // Larger expo than supported. + // println("dns: F5 primes (or larger) are not supported") + return nil + } + pubkey.E = int(expo) + + pubkey.N.SetBytes(keybuf[keyoff+int(explen):]) + return pubkey +} + +// publicKeyECDSA returns the Curve public key from the DNSKEY record. +func (k *DNSKEY) publicKeyECDSA() *ecdsa.PublicKey { + keybuf, err := fromBase64([]byte(k.PublicKey)) + if err != nil { + return nil + } + pubkey := new(ecdsa.PublicKey) + switch k.Algorithm { + case ECDSAP256SHA256: + pubkey.Curve = elliptic.P256() + if len(keybuf) != 64 { + // wrongly encoded key + return nil + } + case ECDSAP384SHA384: + pubkey.Curve = elliptic.P384() + if len(keybuf) != 96 { + // Wrongly encoded key + return nil + } + } + pubkey.X = big.NewInt(0) + pubkey.X.SetBytes(keybuf[:len(keybuf)/2]) + pubkey.Y = big.NewInt(0) + pubkey.Y.SetBytes(keybuf[len(keybuf)/2:]) + return pubkey +} + +func (k *DNSKEY) publicKeyDSA() *dsa.PublicKey { + keybuf, err := fromBase64([]byte(k.PublicKey)) + if err != nil { + return nil + } + if len(keybuf) < 22 { + return nil + } + t, keybuf := int(keybuf[0]), keybuf[1:] + size := 64 + t*8 + q, keybuf := keybuf[:20], keybuf[20:] + if len(keybuf) != 3*size { + return nil + } + p, keybuf := keybuf[:size], keybuf[size:] + g, y := keybuf[:size], keybuf[size:] + pubkey := new(dsa.PublicKey) + pubkey.Parameters.Q = big.NewInt(0).SetBytes(q) + pubkey.Parameters.P = big.NewInt(0).SetBytes(p) + pubkey.Parameters.G = big.NewInt(0).SetBytes(g) + pubkey.Y = big.NewInt(0).SetBytes(y) + return pubkey +} + +type wireSlice [][]byte + +func (p wireSlice) Len() int { return len(p) } +func (p wireSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p wireSlice) Less(i, j int) bool { + _, ioff, _ := UnpackDomainName(p[i], 0) + _, joff, _ := UnpackDomainName(p[j], 0) + return bytes.Compare(p[i][ioff+10:], p[j][joff+10:]) < 0 +} + +// Return the raw signature data. +func rawSignatureData(rrset []RR, s *RRSIG) (buf []byte, err error) { + wires := make(wireSlice, len(rrset)) + for i, r := range rrset { + r1 := r.copy() + r1.Header().Ttl = s.OrigTtl + labels := SplitDomainName(r1.Header().Name) + // 6.2. Canonical RR Form. (4) - wildcards + if len(labels) > int(s.Labels) { + // Wildcard + r1.Header().Name = "*." + strings.Join(labels[len(labels)-int(s.Labels):], ".") + "." + } + // RFC 4034: 6.2. Canonical RR Form. (2) - domain name to lowercase + r1.Header().Name = strings.ToLower(r1.Header().Name) + // 6.2. Canonical RR Form. (3) - domain rdata to lowercase. + // NS, MD, MF, CNAME, SOA, MB, MG, MR, PTR, + // HINFO, MINFO, MX, RP, AFSDB, RT, SIG, PX, NXT, NAPTR, KX, + // SRV, DNAME, A6 + // + // RFC 6840 - Clarifications and Implementation Notes for DNS Security (DNSSEC): + // Section 6.2 of [RFC4034] also erroneously lists HINFO as a record + // that needs conversion to lowercase, and twice at that. Since HINFO + // records contain no domain names, they are not subject to case + // conversion. + switch x := r1.(type) { + case *NS: + x.Ns = strings.ToLower(x.Ns) + case *CNAME: + x.Target = strings.ToLower(x.Target) + case *SOA: + x.Ns = strings.ToLower(x.Ns) + x.Mbox = strings.ToLower(x.Mbox) + case *MB: + x.Mb = strings.ToLower(x.Mb) + case *MG: + x.Mg = strings.ToLower(x.Mg) + case *MR: + x.Mr = strings.ToLower(x.Mr) + case *PTR: + x.Ptr = strings.ToLower(x.Ptr) + case *MINFO: + x.Rmail = strings.ToLower(x.Rmail) + x.Email = strings.ToLower(x.Email) + case *MX: + x.Mx = strings.ToLower(x.Mx) + case *NAPTR: + x.Replacement = strings.ToLower(x.Replacement) + case *KX: + x.Exchanger = strings.ToLower(x.Exchanger) + case *SRV: + x.Target = strings.ToLower(x.Target) + case *DNAME: + x.Target = strings.ToLower(x.Target) + } + // 6.2. Canonical RR Form. (5) - origTTL + wire := make([]byte, r1.len()+1) // +1 to be safe(r) + off, err1 := PackRR(r1, wire, 0, nil, false) + if err1 != nil { + return nil, err1 + } + wire = wire[:off] + wires[i] = wire + } + sort.Sort(wires) + for i, wire := range wires { + if i > 0 && bytes.Equal(wire, wires[i-1]) { + continue + } + buf = append(buf, wire...) + } + return buf, nil +} + +func packSigWire(sw *rrsigWireFmt, msg []byte) (int, error) { + // copied from zmsg.go RRSIG packing + off, err := packUint16(sw.TypeCovered, msg, 0) + if err != nil { + return off, err + } + off, err = packUint8(sw.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(sw.Labels, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(sw.OrigTtl, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(sw.Expiration, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(sw.Inception, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(sw.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(sw.SignerName, msg, off, nil, false) + if err != nil { + return off, err + } + return off, nil +} + +func packKeyWire(dw *dnskeyWireFmt, msg []byte) (int, error) { + // copied from zmsg.go DNSKEY packing + off, err := packUint16(dw.Flags, msg, 0) + if err != nil { + return off, err + } + off, err = packUint8(dw.Protocol, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(dw.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase64(dw.PublicKey, msg, off) + if err != nil { + return off, err + } + return off, nil +} diff --git a/vendor/github.com/miekg/dns/dnssec_keygen.go b/vendor/github.com/miekg/dns/dnssec_keygen.go new file mode 100644 index 00000000..5e4b7741 --- /dev/null +++ b/vendor/github.com/miekg/dns/dnssec_keygen.go @@ -0,0 +1,156 @@ +package dns + +import ( + "crypto" + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "math/big" +) + +// Generate generates a DNSKEY of the given bit size. +// The public part is put inside the DNSKEY record. +// The Algorithm in the key must be set as this will define +// what kind of DNSKEY will be generated. +// The ECDSA algorithms imply a fixed keysize, in that case +// bits should be set to the size of the algorithm. +func (k *DNSKEY) Generate(bits int) (crypto.PrivateKey, error) { + switch k.Algorithm { + case DSA, DSANSEC3SHA1: + if bits != 1024 { + return nil, ErrKeySize + } + case RSAMD5, RSASHA1, RSASHA256, RSASHA1NSEC3SHA1: + if bits < 512 || bits > 4096 { + return nil, ErrKeySize + } + case RSASHA512: + if bits < 1024 || bits > 4096 { + return nil, ErrKeySize + } + case ECDSAP256SHA256: + if bits != 256 { + return nil, ErrKeySize + } + case ECDSAP384SHA384: + if bits != 384 { + return nil, ErrKeySize + } + } + + switch k.Algorithm { + case DSA, DSANSEC3SHA1: + params := new(dsa.Parameters) + if err := dsa.GenerateParameters(params, rand.Reader, dsa.L1024N160); err != nil { + return nil, err + } + priv := new(dsa.PrivateKey) + priv.PublicKey.Parameters = *params + err := dsa.GenerateKey(priv, rand.Reader) + if err != nil { + return nil, err + } + k.setPublicKeyDSA(params.Q, params.P, params.G, priv.PublicKey.Y) + return priv, nil + case RSAMD5, RSASHA1, RSASHA256, RSASHA512, RSASHA1NSEC3SHA1: + priv, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return nil, err + } + k.setPublicKeyRSA(priv.PublicKey.E, priv.PublicKey.N) + return priv, nil + case ECDSAP256SHA256, ECDSAP384SHA384: + var c elliptic.Curve + switch k.Algorithm { + case ECDSAP256SHA256: + c = elliptic.P256() + case ECDSAP384SHA384: + c = elliptic.P384() + } + priv, err := ecdsa.GenerateKey(c, rand.Reader) + if err != nil { + return nil, err + } + k.setPublicKeyECDSA(priv.PublicKey.X, priv.PublicKey.Y) + return priv, nil + default: + return nil, ErrAlg + } +} + +// Set the public key (the value E and N) +func (k *DNSKEY) setPublicKeyRSA(_E int, _N *big.Int) bool { + if _E == 0 || _N == nil { + return false + } + buf := exponentToBuf(_E) + buf = append(buf, _N.Bytes()...) + k.PublicKey = toBase64(buf) + return true +} + +// Set the public key for Elliptic Curves +func (k *DNSKEY) setPublicKeyECDSA(_X, _Y *big.Int) bool { + if _X == nil || _Y == nil { + return false + } + var intlen int + switch k.Algorithm { + case ECDSAP256SHA256: + intlen = 32 + case ECDSAP384SHA384: + intlen = 48 + } + k.PublicKey = toBase64(curveToBuf(_X, _Y, intlen)) + return true +} + +// Set the public key for DSA +func (k *DNSKEY) setPublicKeyDSA(_Q, _P, _G, _Y *big.Int) bool { + if _Q == nil || _P == nil || _G == nil || _Y == nil { + return false + } + buf := dsaToBuf(_Q, _P, _G, _Y) + k.PublicKey = toBase64(buf) + return true +} + +// Set the public key (the values E and N) for RSA +// RFC 3110: Section 2. RSA Public KEY Resource Records +func exponentToBuf(_E int) []byte { + var buf []byte + i := big.NewInt(int64(_E)).Bytes() + if len(i) < 256 { + buf = make([]byte, 1, 1+len(i)) + buf[0] = uint8(len(i)) + } else { + buf = make([]byte, 3, 3+len(i)) + buf[0] = 0 + buf[1] = uint8(len(i) >> 8) + buf[2] = uint8(len(i)) + } + buf = append(buf, i...) + return buf +} + +// Set the public key for X and Y for Curve. The two +// values are just concatenated. +func curveToBuf(_X, _Y *big.Int, intlen int) []byte { + buf := intToBytes(_X, intlen) + buf = append(buf, intToBytes(_Y, intlen)...) + return buf +} + +// Set the public key for X and Y for Curve. The two +// values are just concatenated. +func dsaToBuf(_Q, _P, _G, _Y *big.Int) []byte { + t := divRoundUp(divRoundUp(_G.BitLen(), 8)-64, 8) + buf := []byte{byte(t)} + buf = append(buf, intToBytes(_Q, 20)...) + buf = append(buf, intToBytes(_P, 64+t*8)...) + buf = append(buf, intToBytes(_G, 64+t*8)...) + buf = append(buf, intToBytes(_Y, 64+t*8)...) + return buf +} diff --git a/vendor/github.com/miekg/dns/dnssec_keyscan.go b/vendor/github.com/miekg/dns/dnssec_keyscan.go new file mode 100644 index 00000000..4f8d830b --- /dev/null +++ b/vendor/github.com/miekg/dns/dnssec_keyscan.go @@ -0,0 +1,249 @@ +package dns + +import ( + "crypto" + "crypto/dsa" + "crypto/ecdsa" + "crypto/rsa" + "io" + "math/big" + "strconv" + "strings" +) + +// NewPrivateKey returns a PrivateKey by parsing the string s. +// s should be in the same form of the BIND private key files. +func (k *DNSKEY) NewPrivateKey(s string) (crypto.PrivateKey, error) { + if s == "" || s[len(s)-1] != '\n' { // We need a closing newline + return k.ReadPrivateKey(strings.NewReader(s+"\n"), "") + } + return k.ReadPrivateKey(strings.NewReader(s), "") +} + +// ReadPrivateKey reads a private key from the io.Reader q. The string file is +// only used in error reporting. +// The public key must be known, because some cryptographic algorithms embed +// the public inside the privatekey. +func (k *DNSKEY) ReadPrivateKey(q io.Reader, file string) (crypto.PrivateKey, error) { + m, err := parseKey(q, file) + if m == nil { + return nil, err + } + if _, ok := m["private-key-format"]; !ok { + return nil, ErrPrivKey + } + if m["private-key-format"] != "v1.2" && m["private-key-format"] != "v1.3" { + return nil, ErrPrivKey + } + // TODO(mg): check if the pubkey matches the private key + algo, err := strconv.ParseUint(strings.SplitN(m["algorithm"], " ", 2)[0], 10, 8) + if err != nil { + return nil, ErrPrivKey + } + switch uint8(algo) { + case DSA: + priv, err := readPrivateKeyDSA(m) + if err != nil { + return nil, err + } + pub := k.publicKeyDSA() + if pub == nil { + return nil, ErrKey + } + priv.PublicKey = *pub + return priv, nil + case RSAMD5: + fallthrough + case RSASHA1: + fallthrough + case RSASHA1NSEC3SHA1: + fallthrough + case RSASHA256: + fallthrough + case RSASHA512: + priv, err := readPrivateKeyRSA(m) + if err != nil { + return nil, err + } + pub := k.publicKeyRSA() + if pub == nil { + return nil, ErrKey + } + priv.PublicKey = *pub + return priv, nil + case ECCGOST: + return nil, ErrPrivKey + case ECDSAP256SHA256: + fallthrough + case ECDSAP384SHA384: + priv, err := readPrivateKeyECDSA(m) + if err != nil { + return nil, err + } + pub := k.publicKeyECDSA() + if pub == nil { + return nil, ErrKey + } + priv.PublicKey = *pub + return priv, nil + default: + return nil, ErrPrivKey + } +} + +// Read a private key (file) string and create a public key. Return the private key. +func readPrivateKeyRSA(m map[string]string) (*rsa.PrivateKey, error) { + p := new(rsa.PrivateKey) + p.Primes = []*big.Int{nil, nil} + for k, v := range m { + switch k { + case "modulus", "publicexponent", "privateexponent", "prime1", "prime2": + v1, err := fromBase64([]byte(v)) + if err != nil { + return nil, err + } + switch k { + case "modulus": + p.PublicKey.N = big.NewInt(0) + p.PublicKey.N.SetBytes(v1) + case "publicexponent": + i := big.NewInt(0) + i.SetBytes(v1) + p.PublicKey.E = int(i.Int64()) // int64 should be large enough + case "privateexponent": + p.D = big.NewInt(0) + p.D.SetBytes(v1) + case "prime1": + p.Primes[0] = big.NewInt(0) + p.Primes[0].SetBytes(v1) + case "prime2": + p.Primes[1] = big.NewInt(0) + p.Primes[1].SetBytes(v1) + } + case "exponent1", "exponent2", "coefficient": + // not used in Go (yet) + case "created", "publish", "activate": + // not used in Go (yet) + } + } + return p, nil +} + +func readPrivateKeyDSA(m map[string]string) (*dsa.PrivateKey, error) { + p := new(dsa.PrivateKey) + p.X = big.NewInt(0) + for k, v := range m { + switch k { + case "private_value(x)": + v1, err := fromBase64([]byte(v)) + if err != nil { + return nil, err + } + p.X.SetBytes(v1) + case "created", "publish", "activate": + /* not used in Go (yet) */ + } + } + return p, nil +} + +func readPrivateKeyECDSA(m map[string]string) (*ecdsa.PrivateKey, error) { + p := new(ecdsa.PrivateKey) + p.D = big.NewInt(0) + // TODO: validate that the required flags are present + for k, v := range m { + switch k { + case "privatekey": + v1, err := fromBase64([]byte(v)) + if err != nil { + return nil, err + } + p.D.SetBytes(v1) + case "created", "publish", "activate": + /* not used in Go (yet) */ + } + } + return p, nil +} + +// parseKey reads a private key from r. It returns a map[string]string, +// with the key-value pairs, or an error when the file is not correct. +func parseKey(r io.Reader, file string) (map[string]string, error) { + s := scanInit(r) + m := make(map[string]string) + c := make(chan lex) + k := "" + // Start the lexer + go klexer(s, c) + for l := range c { + // It should alternate + switch l.value { + case zKey: + k = l.token + case zValue: + if k == "" { + return nil, &ParseError{file, "no private key seen", l} + } + //println("Setting", strings.ToLower(k), "to", l.token, "b") + m[strings.ToLower(k)] = l.token + k = "" + } + } + return m, nil +} + +// klexer scans the sourcefile and returns tokens on the channel c. +func klexer(s *scan, c chan lex) { + var l lex + str := "" // Hold the current read text + commt := false + key := true + x, err := s.tokenText() + defer close(c) + for err == nil { + l.column = s.position.Column + l.line = s.position.Line + switch x { + case ':': + if commt { + break + } + l.token = str + if key { + l.value = zKey + c <- l + // Next token is a space, eat it + s.tokenText() + key = false + str = "" + } else { + l.value = zValue + } + case ';': + commt = true + case '\n': + if commt { + // Reset a comment + commt = false + } + l.value = zValue + l.token = str + c <- l + str = "" + commt = false + key = true + default: + if commt { + break + } + str += string(x) + } + x, err = s.tokenText() + } + if len(str) > 0 { + // Send remainder + l.token = str + l.value = zValue + c <- l + } +} diff --git a/vendor/github.com/miekg/dns/dnssec_privkey.go b/vendor/github.com/miekg/dns/dnssec_privkey.go new file mode 100644 index 00000000..56f3ea93 --- /dev/null +++ b/vendor/github.com/miekg/dns/dnssec_privkey.go @@ -0,0 +1,85 @@ +package dns + +import ( + "crypto" + "crypto/dsa" + "crypto/ecdsa" + "crypto/rsa" + "math/big" + "strconv" +) + +const format = "Private-key-format: v1.3\n" + +// PrivateKeyString converts a PrivateKey to a string. This string has the same +// format as the private-key-file of BIND9 (Private-key-format: v1.3). +// It needs some info from the key (the algorithm), so its a method of the DNSKEY +// It supports rsa.PrivateKey, ecdsa.PrivateKey and dsa.PrivateKey +func (r *DNSKEY) PrivateKeyString(p crypto.PrivateKey) string { + algorithm := strconv.Itoa(int(r.Algorithm)) + algorithm += " (" + AlgorithmToString[r.Algorithm] + ")" + + switch p := p.(type) { + case *rsa.PrivateKey: + modulus := toBase64(p.PublicKey.N.Bytes()) + e := big.NewInt(int64(p.PublicKey.E)) + publicExponent := toBase64(e.Bytes()) + privateExponent := toBase64(p.D.Bytes()) + prime1 := toBase64(p.Primes[0].Bytes()) + prime2 := toBase64(p.Primes[1].Bytes()) + // Calculate Exponent1/2 and Coefficient as per: http://en.wikipedia.org/wiki/RSA#Using_the_Chinese_remainder_algorithm + // and from: http://code.google.com/p/go/issues/detail?id=987 + one := big.NewInt(1) + p1 := big.NewInt(0).Sub(p.Primes[0], one) + q1 := big.NewInt(0).Sub(p.Primes[1], one) + exp1 := big.NewInt(0).Mod(p.D, p1) + exp2 := big.NewInt(0).Mod(p.D, q1) + coeff := big.NewInt(0).ModInverse(p.Primes[1], p.Primes[0]) + + exponent1 := toBase64(exp1.Bytes()) + exponent2 := toBase64(exp2.Bytes()) + coefficient := toBase64(coeff.Bytes()) + + return format + + "Algorithm: " + algorithm + "\n" + + "Modulus: " + modulus + "\n" + + "PublicExponent: " + publicExponent + "\n" + + "PrivateExponent: " + privateExponent + "\n" + + "Prime1: " + prime1 + "\n" + + "Prime2: " + prime2 + "\n" + + "Exponent1: " + exponent1 + "\n" + + "Exponent2: " + exponent2 + "\n" + + "Coefficient: " + coefficient + "\n" + + case *ecdsa.PrivateKey: + var intlen int + switch r.Algorithm { + case ECDSAP256SHA256: + intlen = 32 + case ECDSAP384SHA384: + intlen = 48 + } + private := toBase64(intToBytes(p.D, intlen)) + return format + + "Algorithm: " + algorithm + "\n" + + "PrivateKey: " + private + "\n" + + case *dsa.PrivateKey: + T := divRoundUp(divRoundUp(p.PublicKey.Parameters.G.BitLen(), 8)-64, 8) + prime := toBase64(intToBytes(p.PublicKey.Parameters.P, 64+T*8)) + subprime := toBase64(intToBytes(p.PublicKey.Parameters.Q, 20)) + base := toBase64(intToBytes(p.PublicKey.Parameters.G, 64+T*8)) + priv := toBase64(intToBytes(p.X, 20)) + pub := toBase64(intToBytes(p.PublicKey.Y, 64+T*8)) + return format + + "Algorithm: " + algorithm + "\n" + + "Prime(p): " + prime + "\n" + + "Subprime(q): " + subprime + "\n" + + "Base(g): " + base + "\n" + + "Private_value(x): " + priv + "\n" + + "Public_value(y): " + pub + "\n" + + default: + return "" + } +} diff --git a/vendor/github.com/miekg/dns/doc.go b/vendor/github.com/miekg/dns/doc.go new file mode 100644 index 00000000..e38753d7 --- /dev/null +++ b/vendor/github.com/miekg/dns/doc.go @@ -0,0 +1,251 @@ +/* +Package dns implements a full featured interface to the Domain Name System. +Server- and client-side programming is supported. +The package allows complete control over what is send out to the DNS. The package +API follows the less-is-more principle, by presenting a small, clean interface. + +The package dns supports (asynchronous) querying/replying, incoming/outgoing zone transfers, +TSIG, EDNS0, dynamic updates, notifies and DNSSEC validation/signing. +Note that domain names MUST be fully qualified, before sending them, unqualified +names in a message will result in a packing failure. + +Resource records are native types. They are not stored in wire format. +Basic usage pattern for creating a new resource record: + + r := new(dns.MX) + r.Hdr = dns.RR_Header{Name: "miek.nl.", Rrtype: dns.TypeMX, + Class: dns.ClassINET, Ttl: 3600} + r.Preference = 10 + r.Mx = "mx.miek.nl." + +Or directly from a string: + + mx, err := dns.NewRR("miek.nl. 3600 IN MX 10 mx.miek.nl.") + +Or when the default TTL (3600) and class (IN) suit you: + + mx, err := dns.NewRR("miek.nl. MX 10 mx.miek.nl.") + +Or even: + + mx, err := dns.NewRR("$ORIGIN nl.\nmiek 1H IN MX 10 mx.miek") + +In the DNS messages are exchanged, these messages contain resource +records (sets). Use pattern for creating a message: + + m := new(dns.Msg) + m.SetQuestion("miek.nl.", dns.TypeMX) + +Or when not certain if the domain name is fully qualified: + + m.SetQuestion(dns.Fqdn("miek.nl"), dns.TypeMX) + +The message m is now a message with the question section set to ask +the MX records for the miek.nl. zone. + +The following is slightly more verbose, but more flexible: + + m1 := new(dns.Msg) + m1.Id = dns.Id() + m1.RecursionDesired = true + m1.Question = make([]dns.Question, 1) + m1.Question[0] = dns.Question{"miek.nl.", dns.TypeMX, dns.ClassINET} + +After creating a message it can be send. +Basic use pattern for synchronous querying the DNS at a +server configured on 127.0.0.1 and port 53: + + c := new(dns.Client) + in, rtt, err := c.Exchange(m1, "127.0.0.1:53") + +Suppressing multiple outstanding queries (with the same question, type and +class) is as easy as setting: + + c.SingleInflight = true + +If these "advanced" features are not needed, a simple UDP query can be send, +with: + + in, err := dns.Exchange(m1, "127.0.0.1:53") + +When this functions returns you will get dns message. A dns message consists +out of four sections. +The question section: in.Question, the answer section: in.Answer, +the authority section: in.Ns and the additional section: in.Extra. + +Each of these sections (except the Question section) contain a []RR. Basic +use pattern for accessing the rdata of a TXT RR as the first RR in +the Answer section: + + if t, ok := in.Answer[0].(*dns.TXT); ok { + // do something with t.Txt + } + +Domain Name and TXT Character String Representations + +Both domain names and TXT character strings are converted to presentation +form both when unpacked and when converted to strings. + +For TXT character strings, tabs, carriage returns and line feeds will be +converted to \t, \r and \n respectively. Back slashes and quotations marks +will be escaped. Bytes below 32 and above 127 will be converted to \DDD +form. + +For domain names, in addition to the above rules brackets, periods, +spaces, semicolons and the at symbol are escaped. + +DNSSEC + +DNSSEC (DNS Security Extension) adds a layer of security to the DNS. It +uses public key cryptography to sign resource records. The +public keys are stored in DNSKEY records and the signatures in RRSIG records. + +Requesting DNSSEC information for a zone is done by adding the DO (DNSSEC OK) bit +to a request. + + m := new(dns.Msg) + m.SetEdns0(4096, true) + +Signature generation, signature verification and key generation are all supported. + +DYNAMIC UPDATES + +Dynamic updates reuses the DNS message format, but renames three of +the sections. Question is Zone, Answer is Prerequisite, Authority is +Update, only the Additional is not renamed. See RFC 2136 for the gory details. + +You can set a rather complex set of rules for the existence of absence of +certain resource records or names in a zone to specify if resource records +should be added or removed. The table from RFC 2136 supplemented with the Go +DNS function shows which functions exist to specify the prerequisites. + + 3.2.4 - Table Of Metavalues Used In Prerequisite Section + + CLASS TYPE RDATA Meaning Function + -------------------------------------------------------------- + ANY ANY empty Name is in use dns.NameUsed + ANY rrset empty RRset exists (value indep) dns.RRsetUsed + NONE ANY empty Name is not in use dns.NameNotUsed + NONE rrset empty RRset does not exist dns.RRsetNotUsed + zone rrset rr RRset exists (value dep) dns.Used + +The prerequisite section can also be left empty. +If you have decided on the prerequisites you can tell what RRs should +be added or deleted. The next table shows the options you have and +what functions to call. + + 3.4.2.6 - Table Of Metavalues Used In Update Section + + CLASS TYPE RDATA Meaning Function + --------------------------------------------------------------- + ANY ANY empty Delete all RRsets from name dns.RemoveName + ANY rrset empty Delete an RRset dns.RemoveRRset + NONE rrset rr Delete an RR from RRset dns.Remove + zone rrset rr Add to an RRset dns.Insert + +TRANSACTION SIGNATURE + +An TSIG or transaction signature adds a HMAC TSIG record to each message sent. +The supported algorithms include: HmacMD5, HmacSHA1, HmacSHA256 and HmacSHA512. + +Basic use pattern when querying with a TSIG name "axfr." (note that these key names +must be fully qualified - as they are domain names) and the base64 secret +"so6ZGir4GPAqINNh9U5c3A==": + + c := new(dns.Client) + c.TsigSecret = map[string]string{"axfr.": "so6ZGir4GPAqINNh9U5c3A=="} + m := new(dns.Msg) + m.SetQuestion("miek.nl.", dns.TypeMX) + m.SetTsig("axfr.", dns.HmacMD5, 300, time.Now().Unix()) + ... + // When sending the TSIG RR is calculated and filled in before sending + +When requesting an zone transfer (almost all TSIG usage is when requesting zone transfers), with +TSIG, this is the basic use pattern. In this example we request an AXFR for +miek.nl. with TSIG key named "axfr." and secret "so6ZGir4GPAqINNh9U5c3A==" +and using the server 176.58.119.54: + + t := new(dns.Transfer) + m := new(dns.Msg) + t.TsigSecret = map[string]string{"axfr.": "so6ZGir4GPAqINNh9U5c3A=="} + m.SetAxfr("miek.nl.") + m.SetTsig("axfr.", dns.HmacMD5, 300, time.Now().Unix()) + c, err := t.In(m, "176.58.119.54:53") + for r := range c { ... } + +You can now read the records from the transfer as they come in. Each envelope is checked with TSIG. +If something is not correct an error is returned. + +Basic use pattern validating and replying to a message that has TSIG set. + + server := &dns.Server{Addr: ":53", Net: "udp"} + server.TsigSecret = map[string]string{"axfr.": "so6ZGir4GPAqINNh9U5c3A=="} + go server.ListenAndServe() + dns.HandleFunc(".", handleRequest) + + func handleRequest(w dns.ResponseWriter, r *dns.Msg) { + m := new(dns.Msg) + m.SetReply(r) + if r.IsTsig() != nil { + if w.TsigStatus() == nil { + // *Msg r has an TSIG record and it was validated + m.SetTsig("axfr.", dns.HmacMD5, 300, time.Now().Unix()) + } else { + // *Msg r has an TSIG records and it was not valided + } + } + w.WriteMsg(m) + } + +PRIVATE RRS + +RFC 6895 sets aside a range of type codes for private use. This range +is 65,280 - 65,534 (0xFF00 - 0xFFFE). When experimenting with new Resource Records these +can be used, before requesting an official type code from IANA. + +see http://miek.nl/2014/September/21/idn-and-private-rr-in-go-dns/ for more +information. + +EDNS0 + +EDNS0 is an extension mechanism for the DNS defined in RFC 2671 and updated +by RFC 6891. It defines an new RR type, the OPT RR, which is then completely +abused. +Basic use pattern for creating an (empty) OPT RR: + + o := new(dns.OPT) + o.Hdr.Name = "." // MUST be the root zone, per definition. + o.Hdr.Rrtype = dns.TypeOPT + +The rdata of an OPT RR consists out of a slice of EDNS0 (RFC 6891) +interfaces. Currently only a few have been standardized: EDNS0_NSID +(RFC 5001) and EDNS0_SUBNET (draft-vandergaast-edns-client-subnet-02). Note +that these options may be combined in an OPT RR. +Basic use pattern for a server to check if (and which) options are set: + + // o is a dns.OPT + for _, s := range o.Option { + switch e := s.(type) { + case *dns.EDNS0_NSID: + // do stuff with e.Nsid + case *dns.EDNS0_SUBNET: + // access e.Family, e.Address, etc. + } + } + +SIG(0) + +From RFC 2931: + + SIG(0) provides protection for DNS transactions and requests .... + ... protection for glue records, DNS requests, protection for message headers + on requests and responses, and protection of the overall integrity of a response. + +It works like TSIG, except that SIG(0) uses public key cryptography, instead of the shared +secret approach in TSIG. +Supported algorithms: DSA, ECDSAP256SHA256, ECDSAP384SHA384, RSASHA1, RSASHA256 and +RSASHA512. + +Signing subsequent messages in multi-message sessions is not implemented. +*/ +package dns diff --git a/vendor/github.com/miekg/dns/edns.go b/vendor/github.com/miekg/dns/edns.go new file mode 100644 index 00000000..dbff3714 --- /dev/null +++ b/vendor/github.com/miekg/dns/edns.go @@ -0,0 +1,597 @@ +package dns + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "net" + "strconv" +) + +// EDNS0 Option codes. +const ( + EDNS0LLQ = 0x1 // long lived queries: http://tools.ietf.org/html/draft-sekar-dns-llq-01 + EDNS0UL = 0x2 // update lease draft: http://files.dns-sd.org/draft-sekar-dns-ul.txt + EDNS0NSID = 0x3 // nsid (RFC5001) + EDNS0DAU = 0x5 // DNSSEC Algorithm Understood + EDNS0DHU = 0x6 // DS Hash Understood + EDNS0N3U = 0x7 // NSEC3 Hash Understood + EDNS0SUBNET = 0x8 // client-subnet (RFC6891) + EDNS0EXPIRE = 0x9 // EDNS0 expire + EDNS0COOKIE = 0xa // EDNS0 Cookie + EDNS0TCPKEEPALIVE = 0xb // EDNS0 tcp keep alive (RFC7828) + EDNS0SUBNETDRAFT = 0x50fa // Don't use! Use EDNS0SUBNET + EDNS0LOCALSTART = 0xFDE9 // Beginning of range reserved for local/experimental use (RFC6891) + EDNS0LOCALEND = 0xFFFE // End of range reserved for local/experimental use (RFC6891) + _DO = 1 << 15 // dnssec ok +) + +// OPT is the EDNS0 RR appended to messages to convey extra (meta) information. +// See RFC 6891. +type OPT struct { + Hdr RR_Header + Option []EDNS0 `dns:"opt"` +} + +func (rr *OPT) String() string { + s := "\n;; OPT PSEUDOSECTION:\n; EDNS: version " + strconv.Itoa(int(rr.Version())) + "; " + if rr.Do() { + s += "flags: do; " + } else { + s += "flags: ; " + } + s += "udp: " + strconv.Itoa(int(rr.UDPSize())) + + for _, o := range rr.Option { + switch o.(type) { + case *EDNS0_NSID: + s += "\n; NSID: " + o.String() + h, e := o.pack() + var r string + if e == nil { + for _, c := range h { + r += "(" + string(c) + ")" + } + s += " " + r + } + case *EDNS0_SUBNET: + s += "\n; SUBNET: " + o.String() + if o.(*EDNS0_SUBNET).DraftOption { + s += " (draft)" + } + case *EDNS0_COOKIE: + s += "\n; COOKIE: " + o.String() + case *EDNS0_UL: + s += "\n; UPDATE LEASE: " + o.String() + case *EDNS0_LLQ: + s += "\n; LONG LIVED QUERIES: " + o.String() + case *EDNS0_DAU: + s += "\n; DNSSEC ALGORITHM UNDERSTOOD: " + o.String() + case *EDNS0_DHU: + s += "\n; DS HASH UNDERSTOOD: " + o.String() + case *EDNS0_N3U: + s += "\n; NSEC3 HASH UNDERSTOOD: " + o.String() + case *EDNS0_LOCAL: + s += "\n; LOCAL OPT: " + o.String() + } + } + return s +} + +func (rr *OPT) len() int { + l := rr.Hdr.len() + for i := 0; i < len(rr.Option); i++ { + l += 4 // Account for 2-byte option code and 2-byte option length. + lo, _ := rr.Option[i].pack() + l += len(lo) + } + return l +} + +// return the old value -> delete SetVersion? + +// Version returns the EDNS version used. Only zero is defined. +func (rr *OPT) Version() uint8 { + return uint8((rr.Hdr.Ttl & 0x00FF0000) >> 16) +} + +// SetVersion sets the version of EDNS. This is usually zero. +func (rr *OPT) SetVersion(v uint8) { + rr.Hdr.Ttl = rr.Hdr.Ttl&0xFF00FFFF | (uint32(v) << 16) +} + +// ExtendedRcode returns the EDNS extended RCODE field (the upper 8 bits of the TTL). +func (rr *OPT) ExtendedRcode() int { + return int((rr.Hdr.Ttl&0xFF000000)>>24) + 15 +} + +// SetExtendedRcode sets the EDNS extended RCODE field. +func (rr *OPT) SetExtendedRcode(v uint8) { + if v < RcodeBadVers { // Smaller than 16.. Use the 4 bits you have! + return + } + rr.Hdr.Ttl = rr.Hdr.Ttl&0x00FFFFFF | (uint32(v-15) << 24) +} + +// UDPSize returns the UDP buffer size. +func (rr *OPT) UDPSize() uint16 { + return rr.Hdr.Class +} + +// SetUDPSize sets the UDP buffer size. +func (rr *OPT) SetUDPSize(size uint16) { + rr.Hdr.Class = size +} + +// Do returns the value of the DO (DNSSEC OK) bit. +func (rr *OPT) Do() bool { + return rr.Hdr.Ttl&_DO == _DO +} + +// SetDo sets the DO (DNSSEC OK) bit. +// If we pass an argument, set the DO bit to that value. +// It is possible to pass 2 or more arguments. Any arguments after the 1st is silently ignored. +func (rr *OPT) SetDo(do ...bool) { + if len(do) == 1 { + if do[0] { + rr.Hdr.Ttl |= _DO + } else { + rr.Hdr.Ttl &^= _DO + } + } else { + rr.Hdr.Ttl |= _DO + } +} + +// EDNS0 defines an EDNS0 Option. An OPT RR can have multiple options appended to it. +type EDNS0 interface { + // Option returns the option code for the option. + Option() uint16 + // pack returns the bytes of the option data. + pack() ([]byte, error) + // unpack sets the data as found in the buffer. Is also sets + // the length of the slice as the length of the option data. + unpack([]byte) error + // String returns the string representation of the option. + String() string +} + +// EDNS0_NSID option is used to retrieve a nameserver +// identifier. When sending a request Nsid must be set to the empty string +// The identifier is an opaque string encoded as hex. +// Basic use pattern for creating an nsid option: +// +// o := new(dns.OPT) +// o.Hdr.Name = "." +// o.Hdr.Rrtype = dns.TypeOPT +// e := new(dns.EDNS0_NSID) +// e.Code = dns.EDNS0NSID +// e.Nsid = "AA" +// o.Option = append(o.Option, e) +type EDNS0_NSID struct { + Code uint16 // Always EDNS0NSID + Nsid string // This string needs to be hex encoded +} + +func (e *EDNS0_NSID) pack() ([]byte, error) { + h, err := hex.DecodeString(e.Nsid) + if err != nil { + return nil, err + } + return h, nil +} + +func (e *EDNS0_NSID) Option() uint16 { return EDNS0NSID } +func (e *EDNS0_NSID) unpack(b []byte) error { e.Nsid = hex.EncodeToString(b); return nil } +func (e *EDNS0_NSID) String() string { return string(e.Nsid) } + +// EDNS0_SUBNET is the subnet option that is used to give the remote nameserver +// an idea of where the client lives. It can then give back a different +// answer depending on the location or network topology. +// Basic use pattern for creating an subnet option: +// +// o := new(dns.OPT) +// o.Hdr.Name = "." +// o.Hdr.Rrtype = dns.TypeOPT +// e := new(dns.EDNS0_SUBNET) +// e.Code = dns.EDNS0SUBNET +// e.Family = 1 // 1 for IPv4 source address, 2 for IPv6 +// e.SourceNetmask = 32 // 32 for IPV4, 128 for IPv6 +// e.SourceScope = 0 +// e.Address = net.ParseIP("127.0.0.1").To4() // for IPv4 +// // e.Address = net.ParseIP("2001:7b8:32a::2") // for IPV6 +// o.Option = append(o.Option, e) +// +// Note: the spec (draft-ietf-dnsop-edns-client-subnet-00) has some insane logic +// for which netmask applies to the address. This code will parse all the +// available bits when unpacking (up to optlen). When packing it will apply +// SourceNetmask. If you need more advanced logic, patches welcome and good luck. +type EDNS0_SUBNET struct { + Code uint16 // Always EDNS0SUBNET + Family uint16 // 1 for IP, 2 for IP6 + SourceNetmask uint8 + SourceScope uint8 + Address net.IP + DraftOption bool // Set to true if using the old (0x50fa) option code +} + +func (e *EDNS0_SUBNET) Option() uint16 { + if e.DraftOption { + return EDNS0SUBNETDRAFT + } + return EDNS0SUBNET +} + +func (e *EDNS0_SUBNET) pack() ([]byte, error) { + b := make([]byte, 4) + binary.BigEndian.PutUint16(b[0:], e.Family) + b[2] = e.SourceNetmask + b[3] = e.SourceScope + switch e.Family { + case 1: + if e.SourceNetmask > net.IPv4len*8 { + return nil, errors.New("dns: bad netmask") + } + if len(e.Address.To4()) != net.IPv4len { + return nil, errors.New("dns: bad address") + } + ip := e.Address.To4().Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv4len*8)) + needLength := (e.SourceNetmask + 8 - 1) / 8 // division rounding up + b = append(b, ip[:needLength]...) + case 2: + if e.SourceNetmask > net.IPv6len*8 { + return nil, errors.New("dns: bad netmask") + } + if len(e.Address) != net.IPv6len { + return nil, errors.New("dns: bad address") + } + ip := e.Address.Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv6len*8)) + needLength := (e.SourceNetmask + 8 - 1) / 8 // division rounding up + b = append(b, ip[:needLength]...) + default: + return nil, errors.New("dns: bad address family") + } + return b, nil +} + +func (e *EDNS0_SUBNET) unpack(b []byte) error { + if len(b) < 4 { + return ErrBuf + } + e.Family = binary.BigEndian.Uint16(b) + e.SourceNetmask = b[2] + e.SourceScope = b[3] + switch e.Family { + case 1: + if e.SourceNetmask > net.IPv4len*8 || e.SourceScope > net.IPv4len*8 { + return errors.New("dns: bad netmask") + } + addr := make([]byte, net.IPv4len) + for i := 0; i < net.IPv4len && 4+i < len(b); i++ { + addr[i] = b[4+i] + } + e.Address = net.IPv4(addr[0], addr[1], addr[2], addr[3]) + case 2: + if e.SourceNetmask > net.IPv6len*8 || e.SourceScope > net.IPv6len*8 { + return errors.New("dns: bad netmask") + } + addr := make([]byte, net.IPv6len) + for i := 0; i < net.IPv6len && 4+i < len(b); i++ { + addr[i] = b[4+i] + } + e.Address = net.IP{addr[0], addr[1], addr[2], addr[3], addr[4], + addr[5], addr[6], addr[7], addr[8], addr[9], addr[10], + addr[11], addr[12], addr[13], addr[14], addr[15]} + default: + return errors.New("dns: bad address family") + } + return nil +} + +func (e *EDNS0_SUBNET) String() (s string) { + if e.Address == nil { + s = "" + } else if e.Address.To4() != nil { + s = e.Address.String() + } else { + s = "[" + e.Address.String() + "]" + } + s += "/" + strconv.Itoa(int(e.SourceNetmask)) + "/" + strconv.Itoa(int(e.SourceScope)) + return +} + +// The EDNS0_COOKIE option is used to add a DNS Cookie to a message. +// +// o := new(dns.OPT) +// o.Hdr.Name = "." +// o.Hdr.Rrtype = dns.TypeOPT +// e := new(dns.EDNS0_COOKIE) +// e.Code = dns.EDNS0COOKIE +// e.Cookie = "24a5ac.." +// o.Option = append(o.Option, e) +// +// The Cookie field consists out of a client cookie (RFC 7873 Section 4), that is +// always 8 bytes. It may then optionally be followed by the server cookie. The server +// cookie is of variable length, 8 to a maximum of 32 bytes. In other words: +// +// cCookie := o.Cookie[:16] +// sCookie := o.Cookie[16:] +// +// There is no guarantee that the Cookie string has a specific length. +type EDNS0_COOKIE struct { + Code uint16 // Always EDNS0COOKIE + Cookie string // Hex-encoded cookie data +} + +func (e *EDNS0_COOKIE) pack() ([]byte, error) { + h, err := hex.DecodeString(e.Cookie) + if err != nil { + return nil, err + } + return h, nil +} + +func (e *EDNS0_COOKIE) Option() uint16 { return EDNS0COOKIE } +func (e *EDNS0_COOKIE) unpack(b []byte) error { e.Cookie = hex.EncodeToString(b); return nil } +func (e *EDNS0_COOKIE) String() string { return e.Cookie } + +// The EDNS0_UL (Update Lease) (draft RFC) option is used to tell the server to set +// an expiration on an update RR. This is helpful for clients that cannot clean +// up after themselves. This is a draft RFC and more information can be found at +// http://files.dns-sd.org/draft-sekar-dns-ul.txt +// +// o := new(dns.OPT) +// o.Hdr.Name = "." +// o.Hdr.Rrtype = dns.TypeOPT +// e := new(dns.EDNS0_UL) +// e.Code = dns.EDNS0UL +// e.Lease = 120 // in seconds +// o.Option = append(o.Option, e) +type EDNS0_UL struct { + Code uint16 // Always EDNS0UL + Lease uint32 +} + +func (e *EDNS0_UL) Option() uint16 { return EDNS0UL } +func (e *EDNS0_UL) String() string { return strconv.FormatUint(uint64(e.Lease), 10) } + +// Copied: http://golang.org/src/pkg/net/dnsmsg.go +func (e *EDNS0_UL) pack() ([]byte, error) { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, e.Lease) + return b, nil +} + +func (e *EDNS0_UL) unpack(b []byte) error { + if len(b) < 4 { + return ErrBuf + } + e.Lease = binary.BigEndian.Uint32(b) + return nil +} + +// EDNS0_LLQ stands for Long Lived Queries: http://tools.ietf.org/html/draft-sekar-dns-llq-01 +// Implemented for completeness, as the EDNS0 type code is assigned. +type EDNS0_LLQ struct { + Code uint16 // Always EDNS0LLQ + Version uint16 + Opcode uint16 + Error uint16 + Id uint64 + LeaseLife uint32 +} + +func (e *EDNS0_LLQ) Option() uint16 { return EDNS0LLQ } + +func (e *EDNS0_LLQ) pack() ([]byte, error) { + b := make([]byte, 18) + binary.BigEndian.PutUint16(b[0:], e.Version) + binary.BigEndian.PutUint16(b[2:], e.Opcode) + binary.BigEndian.PutUint16(b[4:], e.Error) + binary.BigEndian.PutUint64(b[6:], e.Id) + binary.BigEndian.PutUint32(b[14:], e.LeaseLife) + return b, nil +} + +func (e *EDNS0_LLQ) unpack(b []byte) error { + if len(b) < 18 { + return ErrBuf + } + e.Version = binary.BigEndian.Uint16(b[0:]) + e.Opcode = binary.BigEndian.Uint16(b[2:]) + e.Error = binary.BigEndian.Uint16(b[4:]) + e.Id = binary.BigEndian.Uint64(b[6:]) + e.LeaseLife = binary.BigEndian.Uint32(b[14:]) + return nil +} + +func (e *EDNS0_LLQ) String() string { + s := strconv.FormatUint(uint64(e.Version), 10) + " " + strconv.FormatUint(uint64(e.Opcode), 10) + + " " + strconv.FormatUint(uint64(e.Error), 10) + " " + strconv.FormatUint(uint64(e.Id), 10) + + " " + strconv.FormatUint(uint64(e.LeaseLife), 10) + return s +} + +type EDNS0_DAU struct { + Code uint16 // Always EDNS0DAU + AlgCode []uint8 +} + +func (e *EDNS0_DAU) Option() uint16 { return EDNS0DAU } +func (e *EDNS0_DAU) pack() ([]byte, error) { return e.AlgCode, nil } +func (e *EDNS0_DAU) unpack(b []byte) error { e.AlgCode = b; return nil } + +func (e *EDNS0_DAU) String() string { + s := "" + for i := 0; i < len(e.AlgCode); i++ { + if a, ok := AlgorithmToString[e.AlgCode[i]]; ok { + s += " " + a + } else { + s += " " + strconv.Itoa(int(e.AlgCode[i])) + } + } + return s +} + +type EDNS0_DHU struct { + Code uint16 // Always EDNS0DHU + AlgCode []uint8 +} + +func (e *EDNS0_DHU) Option() uint16 { return EDNS0DHU } +func (e *EDNS0_DHU) pack() ([]byte, error) { return e.AlgCode, nil } +func (e *EDNS0_DHU) unpack(b []byte) error { e.AlgCode = b; return nil } + +func (e *EDNS0_DHU) String() string { + s := "" + for i := 0; i < len(e.AlgCode); i++ { + if a, ok := HashToString[e.AlgCode[i]]; ok { + s += " " + a + } else { + s += " " + strconv.Itoa(int(e.AlgCode[i])) + } + } + return s +} + +type EDNS0_N3U struct { + Code uint16 // Always EDNS0N3U + AlgCode []uint8 +} + +func (e *EDNS0_N3U) Option() uint16 { return EDNS0N3U } +func (e *EDNS0_N3U) pack() ([]byte, error) { return e.AlgCode, nil } +func (e *EDNS0_N3U) unpack(b []byte) error { e.AlgCode = b; return nil } + +func (e *EDNS0_N3U) String() string { + // Re-use the hash map + s := "" + for i := 0; i < len(e.AlgCode); i++ { + if a, ok := HashToString[e.AlgCode[i]]; ok { + s += " " + a + } else { + s += " " + strconv.Itoa(int(e.AlgCode[i])) + } + } + return s +} + +type EDNS0_EXPIRE struct { + Code uint16 // Always EDNS0EXPIRE + Expire uint32 +} + +func (e *EDNS0_EXPIRE) Option() uint16 { return EDNS0EXPIRE } +func (e *EDNS0_EXPIRE) String() string { return strconv.FormatUint(uint64(e.Expire), 10) } + +func (e *EDNS0_EXPIRE) pack() ([]byte, error) { + b := make([]byte, 4) + b[0] = byte(e.Expire >> 24) + b[1] = byte(e.Expire >> 16) + b[2] = byte(e.Expire >> 8) + b[3] = byte(e.Expire) + return b, nil +} + +func (e *EDNS0_EXPIRE) unpack(b []byte) error { + if len(b) < 4 { + return ErrBuf + } + e.Expire = binary.BigEndian.Uint32(b) + return nil +} + +// The EDNS0_LOCAL option is used for local/experimental purposes. The option +// code is recommended to be within the range [EDNS0LOCALSTART, EDNS0LOCALEND] +// (RFC6891), although any unassigned code can actually be used. The content of +// the option is made available in Data, unaltered. +// Basic use pattern for creating a local option: +// +// o := new(dns.OPT) +// o.Hdr.Name = "." +// o.Hdr.Rrtype = dns.TypeOPT +// e := new(dns.EDNS0_LOCAL) +// e.Code = dns.EDNS0LOCALSTART +// e.Data = []byte{72, 82, 74} +// o.Option = append(o.Option, e) +type EDNS0_LOCAL struct { + Code uint16 + Data []byte +} + +func (e *EDNS0_LOCAL) Option() uint16 { return e.Code } +func (e *EDNS0_LOCAL) String() string { + return strconv.FormatInt(int64(e.Code), 10) + ":0x" + hex.EncodeToString(e.Data) +} + +func (e *EDNS0_LOCAL) pack() ([]byte, error) { + b := make([]byte, len(e.Data)) + copied := copy(b, e.Data) + if copied != len(e.Data) { + return nil, ErrBuf + } + return b, nil +} + +func (e *EDNS0_LOCAL) unpack(b []byte) error { + e.Data = make([]byte, len(b)) + copied := copy(e.Data, b) + if copied != len(b) { + return ErrBuf + } + return nil +} + +// EDNS0_TCP_KEEPALIVE is an EDNS0 option that instructs the server to keep +// the TCP connection alive. See RFC 7828. +type EDNS0_TCP_KEEPALIVE struct { + Code uint16 // Always EDNSTCPKEEPALIVE + Length uint16 // the value 0 if the TIMEOUT is omitted, the value 2 if it is present; + Timeout uint16 // an idle timeout value for the TCP connection, specified in units of 100 milliseconds, encoded in network byte order. +} + +func (e *EDNS0_TCP_KEEPALIVE) Option() uint16 { return EDNS0TCPKEEPALIVE } + +func (e *EDNS0_TCP_KEEPALIVE) pack() ([]byte, error) { + if e.Timeout != 0 && e.Length != 2 { + return nil, errors.New("dns: timeout specified but length is not 2") + } + if e.Timeout == 0 && e.Length != 0 { + return nil, errors.New("dns: timeout not specified but length is not 0") + } + b := make([]byte, 4+e.Length) + binary.BigEndian.PutUint16(b[0:], e.Code) + binary.BigEndian.PutUint16(b[2:], e.Length) + if e.Length == 2 { + binary.BigEndian.PutUint16(b[4:], e.Timeout) + } + return b, nil +} + +func (e *EDNS0_TCP_KEEPALIVE) unpack(b []byte) error { + if len(b) < 4 { + return ErrBuf + } + e.Length = binary.BigEndian.Uint16(b[2:4]) + if e.Length != 0 && e.Length != 2 { + return errors.New("dns: length mismatch, want 0/2 but got " + strconv.FormatUint(uint64(e.Length), 10)) + } + if e.Length == 2 { + if len(b) < 6 { + return ErrBuf + } + e.Timeout = binary.BigEndian.Uint16(b[4:6]) + } + return nil +} + +func (e *EDNS0_TCP_KEEPALIVE) String() (s string) { + s = "use tcp keep-alive" + if e.Length == 0 { + s += ", timeout omitted" + } else { + s += fmt.Sprintf(", timeout %dms", e.Timeout*100) + } + return +} diff --git a/vendor/github.com/miekg/dns/format.go b/vendor/github.com/miekg/dns/format.go new file mode 100644 index 00000000..3f5303c2 --- /dev/null +++ b/vendor/github.com/miekg/dns/format.go @@ -0,0 +1,87 @@ +package dns + +import ( + "net" + "reflect" + "strconv" +) + +// NumField returns the number of rdata fields r has. +func NumField(r RR) int { + return reflect.ValueOf(r).Elem().NumField() - 1 // Remove RR_Header +} + +// Field returns the rdata field i as a string. Fields are indexed starting from 1. +// RR types that holds slice data, for instance the NSEC type bitmap will return a single +// string where the types are concatenated using a space. +// Accessing non existing fields will cause a panic. +func Field(r RR, i int) string { + if i == 0 { + return "" + } + d := reflect.ValueOf(r).Elem().Field(i) + switch k := d.Kind(); k { + case reflect.String: + return d.String() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.FormatInt(d.Int(), 10) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return strconv.FormatUint(d.Uint(), 10) + case reflect.Slice: + switch reflect.ValueOf(r).Elem().Type().Field(i).Tag { + case `dns:"a"`: + // TODO(miek): Hmm store this as 16 bytes + if d.Len() < net.IPv6len { + return net.IPv4(byte(d.Index(0).Uint()), + byte(d.Index(1).Uint()), + byte(d.Index(2).Uint()), + byte(d.Index(3).Uint())).String() + } + return net.IPv4(byte(d.Index(12).Uint()), + byte(d.Index(13).Uint()), + byte(d.Index(14).Uint()), + byte(d.Index(15).Uint())).String() + case `dns:"aaaa"`: + return net.IP{ + byte(d.Index(0).Uint()), + byte(d.Index(1).Uint()), + byte(d.Index(2).Uint()), + byte(d.Index(3).Uint()), + byte(d.Index(4).Uint()), + byte(d.Index(5).Uint()), + byte(d.Index(6).Uint()), + byte(d.Index(7).Uint()), + byte(d.Index(8).Uint()), + byte(d.Index(9).Uint()), + byte(d.Index(10).Uint()), + byte(d.Index(11).Uint()), + byte(d.Index(12).Uint()), + byte(d.Index(13).Uint()), + byte(d.Index(14).Uint()), + byte(d.Index(15).Uint()), + }.String() + case `dns:"nsec"`: + if d.Len() == 0 { + return "" + } + s := Type(d.Index(0).Uint()).String() + for i := 1; i < d.Len(); i++ { + s += " " + Type(d.Index(i).Uint()).String() + } + return s + default: + // if it does not have a tag its a string slice + fallthrough + case `dns:"txt"`: + if d.Len() == 0 { + return "" + } + s := d.Index(0).String() + for i := 1; i < d.Len(); i++ { + s += " " + d.Index(i).String() + } + return s + } + } + return "" +} diff --git a/vendor/github.com/miekg/dns/generate.go b/vendor/github.com/miekg/dns/generate.go new file mode 100644 index 00000000..e4481a4b --- /dev/null +++ b/vendor/github.com/miekg/dns/generate.go @@ -0,0 +1,159 @@ +package dns + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" +) + +// Parse the $GENERATE statement as used in BIND9 zones. +// See http://www.zytrax.com/books/dns/ch8/generate.html for instance. +// We are called after '$GENERATE '. After which we expect: +// * the range (12-24/2) +// * lhs (ownername) +// * [[ttl][class]] +// * type +// * rhs (rdata) +// But we are lazy here, only the range is parsed *all* occurrences +// of $ after that are interpreted. +// Any error are returned as a string value, the empty string signals +// "no error". +func generate(l lex, c chan lex, t chan *Token, o string) string { + step := 1 + if i := strings.IndexAny(l.token, "/"); i != -1 { + if i+1 == len(l.token) { + return "bad step in $GENERATE range" + } + if s, err := strconv.Atoi(l.token[i+1:]); err == nil { + if s < 0 { + return "bad step in $GENERATE range" + } + step = s + } else { + return "bad step in $GENERATE range" + } + l.token = l.token[:i] + } + sx := strings.SplitN(l.token, "-", 2) + if len(sx) != 2 { + return "bad start-stop in $GENERATE range" + } + start, err := strconv.Atoi(sx[0]) + if err != nil { + return "bad start in $GENERATE range" + } + end, err := strconv.Atoi(sx[1]) + if err != nil { + return "bad stop in $GENERATE range" + } + if end < 0 || start < 0 || end < start { + return "bad range in $GENERATE range" + } + + <-c // _BLANK + // Create a complete new string, which we then parse again. + s := "" +BuildRR: + l = <-c + if l.value != zNewline && l.value != zEOF { + s += l.token + goto BuildRR + } + for i := start; i <= end; i += step { + var ( + escape bool + dom bytes.Buffer + mod string + err error + offset int + ) + + for j := 0; j < len(s); j++ { // No 'range' because we need to jump around + switch s[j] { + case '\\': + if escape { + dom.WriteByte('\\') + escape = false + continue + } + escape = true + case '$': + mod = "%d" + offset = 0 + if escape { + dom.WriteByte('$') + escape = false + continue + } + escape = false + if j+1 >= len(s) { // End of the string + dom.WriteString(fmt.Sprintf(mod, i+offset)) + continue + } else { + if s[j+1] == '$' { + dom.WriteByte('$') + j++ + continue + } + } + // Search for { and } + if s[j+1] == '{' { // Modifier block + sep := strings.Index(s[j+2:], "}") + if sep == -1 { + return "bad modifier in $GENERATE" + } + mod, offset, err = modToPrintf(s[j+2 : j+2+sep]) + if err != nil { + return err.Error() + } + j += 2 + sep // Jump to it + } + dom.WriteString(fmt.Sprintf(mod, i+offset)) + default: + if escape { // Pretty useless here + escape = false + continue + } + dom.WriteByte(s[j]) + } + } + // Re-parse the RR and send it on the current channel t + rx, err := NewRR("$ORIGIN " + o + "\n" + dom.String()) + if err != nil { + return err.Error() + } + t <- &Token{RR: rx} + // Its more efficient to first built the rrlist and then parse it in + // one go! But is this a problem? + } + return "" +} + +// Convert a $GENERATE modifier 0,0,d to something Printf can deal with. +func modToPrintf(s string) (string, int, error) { + xs := strings.SplitN(s, ",", 3) + if len(xs) != 3 { + return "", 0, errors.New("bad modifier in $GENERATE") + } + // xs[0] is offset, xs[1] is width, xs[2] is base + if xs[2] != "o" && xs[2] != "d" && xs[2] != "x" && xs[2] != "X" { + return "", 0, errors.New("bad base in $GENERATE") + } + offset, err := strconv.Atoi(xs[0]) + if err != nil || offset > 255 { + return "", 0, errors.New("bad offset in $GENERATE") + } + width, err := strconv.Atoi(xs[1]) + if err != nil || width > 255 { + return "", offset, errors.New("bad width in $GENERATE") + } + switch { + case width < 0: + return "", offset, errors.New("bad width in $GENERATE") + case width == 0: + return "%" + xs[1] + xs[2], offset, nil + } + return "%0" + xs[1] + xs[2], offset, nil +} diff --git a/vendor/github.com/miekg/dns/labels.go b/vendor/github.com/miekg/dns/labels.go new file mode 100644 index 00000000..9538d9c3 --- /dev/null +++ b/vendor/github.com/miekg/dns/labels.go @@ -0,0 +1,171 @@ +package dns + +import "strings" + +// Holds a bunch of helper functions for dealing with labels. + +// SplitDomainName splits a name string into it's labels. +// www.miek.nl. returns []string{"www", "miek", "nl"} +// .www.miek.nl. returns []string{"", "www", "miek", "nl"}, +// The root label (.) returns nil. Note that using +// strings.Split(s) will work in most cases, but does not handle +// escaped dots (\.) for instance. +// s must be a syntactically valid domain name, see IsDomainName. +func SplitDomainName(s string) (labels []string) { + if len(s) == 0 { + return nil + } + fqdnEnd := 0 // offset of the final '.' or the length of the name + idx := Split(s) + begin := 0 + if s[len(s)-1] == '.' { + fqdnEnd = len(s) - 1 + } else { + fqdnEnd = len(s) + } + + switch len(idx) { + case 0: + return nil + case 1: + // no-op + default: + end := 0 + for i := 1; i < len(idx); i++ { + end = idx[i] + labels = append(labels, s[begin:end-1]) + begin = end + } + } + + labels = append(labels, s[begin:fqdnEnd]) + return labels +} + +// CompareDomainName compares the names s1 and s2 and +// returns how many labels they have in common starting from the *right*. +// The comparison stops at the first inequality. The names are not downcased +// before the comparison. +// +// www.miek.nl. and miek.nl. have two labels in common: miek and nl +// www.miek.nl. and www.bla.nl. have one label in common: nl +// +// s1 and s2 must be syntactically valid domain names. +func CompareDomainName(s1, s2 string) (n int) { + s1, s2 = strings.ToLower(s1), strings.ToLower(s2) + s1 = Fqdn(s1) + s2 = Fqdn(s2) + l1 := Split(s1) + l2 := Split(s2) + + // the first check: root label + if l1 == nil || l2 == nil { + return + } + + j1 := len(l1) - 1 // end + i1 := len(l1) - 2 // start + j2 := len(l2) - 1 + i2 := len(l2) - 2 + // the second check can be done here: last/only label + // before we fall through into the for-loop below + if s1[l1[j1]:] == s2[l2[j2]:] { + n++ + } else { + return + } + for { + if i1 < 0 || i2 < 0 { + break + } + if s1[l1[i1]:l1[j1]] == s2[l2[i2]:l2[j2]] { + n++ + } else { + break + } + j1-- + i1-- + j2-- + i2-- + } + return +} + +// CountLabel counts the the number of labels in the string s. +// s must be a syntactically valid domain name. +func CountLabel(s string) (labels int) { + if s == "." { + return + } + off := 0 + end := false + for { + off, end = NextLabel(s, off) + labels++ + if end { + return + } + } +} + +// Split splits a name s into its label indexes. +// www.miek.nl. returns []int{0, 4, 9}, www.miek.nl also returns []int{0, 4, 9}. +// The root name (.) returns nil. Also see SplitDomainName. +// s must be a syntactically valid domain name. +func Split(s string) []int { + if s == "." { + return nil + } + idx := make([]int, 1, 3) + off := 0 + end := false + + for { + off, end = NextLabel(s, off) + if end { + return idx + } + idx = append(idx, off) + } +} + +// NextLabel returns the index of the start of the next label in the +// string s starting at offset. +// The bool end is true when the end of the string has been reached. +// Also see PrevLabel. +func NextLabel(s string, offset int) (i int, end bool) { + quote := false + for i = offset; i < len(s)-1; i++ { + switch s[i] { + case '\\': + quote = !quote + default: + quote = false + case '.': + if quote { + quote = !quote + continue + } + return i + 1, false + } + } + return i + 1, true +} + +// PrevLabel returns the index of the label when starting from the right and +// jumping n labels to the left. +// The bool start is true when the start of the string has been overshot. +// Also see NextLabel. +func PrevLabel(s string, n int) (i int, start bool) { + if n == 0 { + return len(s), false + } + lab := Split(s) + if lab == nil { + return 0, true + } + if n > len(lab) { + return 0, true + } + return lab[len(lab)-n], false +} diff --git a/vendor/github.com/miekg/dns/msg.go b/vendor/github.com/miekg/dns/msg.go new file mode 100644 index 00000000..a348f7c9 --- /dev/null +++ b/vendor/github.com/miekg/dns/msg.go @@ -0,0 +1,1159 @@ +// DNS packet assembly, see RFC 1035. Converting from - Unpack() - +// and to - Pack() - wire format. +// All the packers and unpackers take a (msg []byte, off int) +// and return (off1 int, ok bool). If they return ok==false, they +// also return off1==len(msg), so that the next unpacker will +// also fail. This lets us avoid checks of ok until the end of a +// packing sequence. + +package dns + +//go:generate go run msg_generate.go +//go:generate go run compress_generate.go + +import ( + crand "crypto/rand" + "encoding/binary" + "fmt" + "math/big" + "math/rand" + "strconv" + "sync" +) + +const ( + maxCompressionOffset = 2 << 13 // We have 14 bits for the compression pointer + maxDomainNameWireOctets = 255 // See RFC 1035 section 2.3.4 +) + +var ( + ErrAlg error = &Error{err: "bad algorithm"} // ErrAlg indicates an error with the (DNSSEC) algorithm. + ErrAuth error = &Error{err: "bad authentication"} // ErrAuth indicates an error in the TSIG authentication. + ErrBuf error = &Error{err: "buffer size too small"} // ErrBuf indicates that the buffer used it too small for the message. + ErrConnEmpty error = &Error{err: "conn has no connection"} // ErrConnEmpty indicates a connection is being uses before it is initialized. + ErrExtendedRcode error = &Error{err: "bad extended rcode"} // ErrExtendedRcode ... + ErrFqdn error = &Error{err: "domain must be fully qualified"} // ErrFqdn indicates that a domain name does not have a closing dot. + ErrId error = &Error{err: "id mismatch"} // ErrId indicates there is a mismatch with the message's ID. + ErrKeyAlg error = &Error{err: "bad key algorithm"} // ErrKeyAlg indicates that the algorithm in the key is not valid. + ErrKey error = &Error{err: "bad key"} + ErrKeySize error = &Error{err: "bad key size"} + ErrLongDomain error = &Error{err: fmt.Sprintf("domain name exceeded %d wire-format octets", maxDomainNameWireOctets)} + ErrNoSig error = &Error{err: "no signature found"} + ErrPrivKey error = &Error{err: "bad private key"} + ErrRcode error = &Error{err: "bad rcode"} + ErrRdata error = &Error{err: "bad rdata"} + ErrRRset error = &Error{err: "bad rrset"} + ErrSecret error = &Error{err: "no secrets defined"} + ErrShortRead error = &Error{err: "short read"} + ErrSig error = &Error{err: "bad signature"} // ErrSig indicates that a signature can not be cryptographically validated. + ErrSoa error = &Error{err: "no SOA"} // ErrSOA indicates that no SOA RR was seen when doing zone transfers. + ErrTime error = &Error{err: "bad time"} // ErrTime indicates a timing error in TSIG authentication. + ErrTruncated error = &Error{err: "failed to unpack truncated message"} // ErrTruncated indicates that we failed to unpack a truncated message. We unpacked as much as we had so Msg can still be used, if desired. +) + +// Id by default, returns a 16 bits random number to be used as a +// message id. The random provided should be good enough. This being a +// variable the function can be reassigned to a custom function. +// For instance, to make it return a static value: +// +// dns.Id = func() uint16 { return 3 } +var Id func() uint16 = id + +var ( + idLock sync.Mutex + idRand *rand.Rand +) + +// id returns a 16 bits random number to be used as a +// message id. The random provided should be good enough. +func id() uint16 { + idLock.Lock() + + if idRand == nil { + // This (partially) works around + // https://github.com/golang/go/issues/11833 by only + // seeding idRand upon the first call to id. + + var seed int64 + var buf [8]byte + + if _, err := crand.Read(buf[:]); err == nil { + seed = int64(binary.LittleEndian.Uint64(buf[:])) + } else { + seed = rand.Int63() + } + + idRand = rand.New(rand.NewSource(seed)) + } + + // The call to idRand.Uint32 must be within the + // mutex lock because *rand.Rand is not safe for + // concurrent use. + // + // There is no added performance overhead to calling + // idRand.Uint32 inside a mutex lock over just + // calling rand.Uint32 as the global math/rand rng + // is internally protected by a sync.Mutex. + id := uint16(idRand.Uint32()) + + idLock.Unlock() + return id +} + +// MsgHdr is a a manually-unpacked version of (id, bits). +type MsgHdr struct { + Id uint16 + Response bool + Opcode int + Authoritative bool + Truncated bool + RecursionDesired bool + RecursionAvailable bool + Zero bool + AuthenticatedData bool + CheckingDisabled bool + Rcode int +} + +// Msg contains the layout of a DNS message. +type Msg struct { + MsgHdr + Compress bool `json:"-"` // If true, the message will be compressed when converted to wire format. + Question []Question // Holds the RR(s) of the question section. + Answer []RR // Holds the RR(s) of the answer section. + Ns []RR // Holds the RR(s) of the authority section. + Extra []RR // Holds the RR(s) of the additional section. +} + +// ClassToString is a maps Classes to strings for each CLASS wire type. +var ClassToString = map[uint16]string{ + ClassINET: "IN", + ClassCSNET: "CS", + ClassCHAOS: "CH", + ClassHESIOD: "HS", + ClassNONE: "NONE", + ClassANY: "ANY", +} + +// OpcodeToString maps Opcodes to strings. +var OpcodeToString = map[int]string{ + OpcodeQuery: "QUERY", + OpcodeIQuery: "IQUERY", + OpcodeStatus: "STATUS", + OpcodeNotify: "NOTIFY", + OpcodeUpdate: "UPDATE", +} + +// RcodeToString maps Rcodes to strings. +var RcodeToString = map[int]string{ + RcodeSuccess: "NOERROR", + RcodeFormatError: "FORMERR", + RcodeServerFailure: "SERVFAIL", + RcodeNameError: "NXDOMAIN", + RcodeNotImplemented: "NOTIMPL", + RcodeRefused: "REFUSED", + RcodeYXDomain: "YXDOMAIN", // See RFC 2136 + RcodeYXRrset: "YXRRSET", + RcodeNXRrset: "NXRRSET", + RcodeNotAuth: "NOTAUTH", + RcodeNotZone: "NOTZONE", + RcodeBadSig: "BADSIG", // Also known as RcodeBadVers, see RFC 6891 + // RcodeBadVers: "BADVERS", + RcodeBadKey: "BADKEY", + RcodeBadTime: "BADTIME", + RcodeBadMode: "BADMODE", + RcodeBadName: "BADNAME", + RcodeBadAlg: "BADALG", + RcodeBadTrunc: "BADTRUNC", + RcodeBadCookie: "BADCOOKIE", +} + +// Domain names are a sequence of counted strings +// split at the dots. They end with a zero-length string. + +// PackDomainName packs a domain name s into msg[off:]. +// If compression is wanted compress must be true and the compression +// map needs to hold a mapping between domain names and offsets +// pointing into msg. +func PackDomainName(s string, msg []byte, off int, compression map[string]int, compress bool) (off1 int, err error) { + off1, _, err = packDomainName(s, msg, off, compression, compress) + return +} + +func packDomainName(s string, msg []byte, off int, compression map[string]int, compress bool) (off1 int, labels int, err error) { + // special case if msg == nil + lenmsg := 256 + if msg != nil { + lenmsg = len(msg) + } + ls := len(s) + if ls == 0 { // Ok, for instance when dealing with update RR without any rdata. + return off, 0, nil + } + // If not fully qualified, error out, but only if msg == nil #ugly + switch { + case msg == nil: + if s[ls-1] != '.' { + s += "." + ls++ + } + case msg != nil: + if s[ls-1] != '.' { + return lenmsg, 0, ErrFqdn + } + } + // Each dot ends a segment of the name. + // We trade each dot byte for a length byte. + // Except for escaped dots (\.), which are normal dots. + // There is also a trailing zero. + + // Compression + nameoffset := -1 + pointer := -1 + // Emit sequence of counted strings, chopping at dots. + begin := 0 + bs := []byte(s) + roBs, bsFresh, escapedDot := s, true, false + for i := 0; i < ls; i++ { + if bs[i] == '\\' { + for j := i; j < ls-1; j++ { + bs[j] = bs[j+1] + } + ls-- + if off+1 > lenmsg { + return lenmsg, labels, ErrBuf + } + // check for \DDD + if i+2 < ls && isDigit(bs[i]) && isDigit(bs[i+1]) && isDigit(bs[i+2]) { + bs[i] = dddToByte(bs[i:]) + for j := i + 1; j < ls-2; j++ { + bs[j] = bs[j+2] + } + ls -= 2 + } + escapedDot = bs[i] == '.' + bsFresh = false + continue + } + + if bs[i] == '.' { + if i > 0 && bs[i-1] == '.' && !escapedDot { + // two dots back to back is not legal + return lenmsg, labels, ErrRdata + } + if i-begin >= 1<<6 { // top two bits of length must be clear + return lenmsg, labels, ErrRdata + } + // off can already (we're in a loop) be bigger than len(msg) + // this happens when a name isn't fully qualified + if off+1 > lenmsg { + return lenmsg, labels, ErrBuf + } + if msg != nil { + msg[off] = byte(i - begin) + } + offset := off + off++ + for j := begin; j < i; j++ { + if off+1 > lenmsg { + return lenmsg, labels, ErrBuf + } + if msg != nil { + msg[off] = bs[j] + } + off++ + } + if compress && !bsFresh { + roBs = string(bs) + bsFresh = true + } + // Don't try to compress '.' + // We should only compress when compress it true, but we should also still pick + // up names that can be used for *future* compression(s). + if compression != nil && roBs[begin:] != "." { + if p, ok := compression[roBs[begin:]]; !ok { + // Only offsets smaller than this can be used. + if offset < maxCompressionOffset { + compression[roBs[begin:]] = offset + } + } else { + // The first hit is the longest matching dname + // keep the pointer offset we get back and store + // the offset of the current name, because that's + // where we need to insert the pointer later + + // If compress is true, we're allowed to compress this dname + if pointer == -1 && compress { + pointer = p // Where to point to + nameoffset = offset // Where to point from + break + } + } + } + labels++ + begin = i + 1 + } + escapedDot = false + } + // Root label is special + if len(bs) == 1 && bs[0] == '.' { + return off, labels, nil + } + // If we did compression and we find something add the pointer here + if pointer != -1 { + // We have two bytes (14 bits) to put the pointer in + // if msg == nil, we will never do compression + binary.BigEndian.PutUint16(msg[nameoffset:], uint16(pointer^0xC000)) + off = nameoffset + 1 + goto End + } + if msg != nil && off < len(msg) { + msg[off] = 0 + } +End: + off++ + return off, labels, nil +} + +// Unpack a domain name. +// In addition to the simple sequences of counted strings above, +// domain names are allowed to refer to strings elsewhere in the +// packet, to avoid repeating common suffixes when returning +// many entries in a single domain. The pointers are marked +// by a length byte with the top two bits set. Ignoring those +// two bits, that byte and the next give a 14 bit offset from msg[0] +// where we should pick up the trail. +// Note that if we jump elsewhere in the packet, +// we return off1 == the offset after the first pointer we found, +// which is where the next record will start. +// In theory, the pointers are only allowed to jump backward. +// We let them jump anywhere and stop jumping after a while. + +// UnpackDomainName unpacks a domain name into a string. +func UnpackDomainName(msg []byte, off int) (string, int, error) { + s := make([]byte, 0, 64) + off1 := 0 + lenmsg := len(msg) + maxLen := maxDomainNameWireOctets + ptr := 0 // number of pointers followed +Loop: + for { + if off >= lenmsg { + return "", lenmsg, ErrBuf + } + c := int(msg[off]) + off++ + switch c & 0xC0 { + case 0x00: + if c == 0x00 { + // end of name + break Loop + } + // literal string + if off+c > lenmsg { + return "", lenmsg, ErrBuf + } + for j := off; j < off+c; j++ { + switch b := msg[j]; b { + case '.', '(', ')', ';', ' ', '@': + fallthrough + case '"', '\\': + s = append(s, '\\', b) + // presentation-format \X escapes add an extra byte + maxLen += 1 + default: + if b < 32 || b >= 127 { // unprintable, use \DDD + var buf [3]byte + bufs := strconv.AppendInt(buf[:0], int64(b), 10) + s = append(s, '\\') + for i := 0; i < 3-len(bufs); i++ { + s = append(s, '0') + } + for _, r := range bufs { + s = append(s, r) + } + // presentation-format \DDD escapes add 3 extra bytes + maxLen += 3 + } else { + s = append(s, b) + } + } + } + s = append(s, '.') + off += c + case 0xC0: + // pointer to somewhere else in msg. + // remember location after first ptr, + // since that's how many bytes we consumed. + // also, don't follow too many pointers -- + // maybe there's a loop. + if off >= lenmsg { + return "", lenmsg, ErrBuf + } + c1 := msg[off] + off++ + if ptr == 0 { + off1 = off + } + if ptr++; ptr > 10 { + return "", lenmsg, &Error{err: "too many compression pointers"} + } + // pointer should guarantee that it advances and points forwards at least + // but the condition on previous three lines guarantees that it's + // at least loop-free + off = (c^0xC0)<<8 | int(c1) + default: + // 0x80 and 0x40 are reserved + return "", lenmsg, ErrRdata + } + } + if ptr == 0 { + off1 = off + } + if len(s) == 0 { + s = []byte(".") + } else if len(s) >= maxLen { + // error if the name is too long, but don't throw it away + return string(s), lenmsg, ErrLongDomain + } + return string(s), off1, nil +} + +func packTxt(txt []string, msg []byte, offset int, tmp []byte) (int, error) { + if len(txt) == 0 { + if offset >= len(msg) { + return offset, ErrBuf + } + msg[offset] = 0 + return offset, nil + } + var err error + for i := range txt { + if len(txt[i]) > len(tmp) { + return offset, ErrBuf + } + offset, err = packTxtString(txt[i], msg, offset, tmp) + if err != nil { + return offset, err + } + } + return offset, nil +} + +func packTxtString(s string, msg []byte, offset int, tmp []byte) (int, error) { + lenByteOffset := offset + if offset >= len(msg) || len(s) > len(tmp) { + return offset, ErrBuf + } + offset++ + bs := tmp[:len(s)] + copy(bs, s) + for i := 0; i < len(bs); i++ { + if len(msg) <= offset { + return offset, ErrBuf + } + if bs[i] == '\\' { + i++ + if i == len(bs) { + break + } + // check for \DDD + if i+2 < len(bs) && isDigit(bs[i]) && isDigit(bs[i+1]) && isDigit(bs[i+2]) { + msg[offset] = dddToByte(bs[i:]) + i += 2 + } else { + msg[offset] = bs[i] + } + } else { + msg[offset] = bs[i] + } + offset++ + } + l := offset - lenByteOffset - 1 + if l > 255 { + return offset, &Error{err: "string exceeded 255 bytes in txt"} + } + msg[lenByteOffset] = byte(l) + return offset, nil +} + +func packOctetString(s string, msg []byte, offset int, tmp []byte) (int, error) { + if offset >= len(msg) || len(s) > len(tmp) { + return offset, ErrBuf + } + bs := tmp[:len(s)] + copy(bs, s) + for i := 0; i < len(bs); i++ { + if len(msg) <= offset { + return offset, ErrBuf + } + if bs[i] == '\\' { + i++ + if i == len(bs) { + break + } + // check for \DDD + if i+2 < len(bs) && isDigit(bs[i]) && isDigit(bs[i+1]) && isDigit(bs[i+2]) { + msg[offset] = dddToByte(bs[i:]) + i += 2 + } else { + msg[offset] = bs[i] + } + } else { + msg[offset] = bs[i] + } + offset++ + } + return offset, nil +} + +func unpackTxt(msg []byte, off0 int) (ss []string, off int, err error) { + off = off0 + var s string + for off < len(msg) && err == nil { + s, off, err = unpackTxtString(msg, off) + if err == nil { + ss = append(ss, s) + } + } + return +} + +func unpackTxtString(msg []byte, offset int) (string, int, error) { + if offset+1 > len(msg) { + return "", offset, &Error{err: "overflow unpacking txt"} + } + l := int(msg[offset]) + if offset+l+1 > len(msg) { + return "", offset, &Error{err: "overflow unpacking txt"} + } + s := make([]byte, 0, l) + for _, b := range msg[offset+1 : offset+1+l] { + switch b { + case '"', '\\': + s = append(s, '\\', b) + default: + if b < 32 || b > 127 { // unprintable + var buf [3]byte + bufs := strconv.AppendInt(buf[:0], int64(b), 10) + s = append(s, '\\') + for i := 0; i < 3-len(bufs); i++ { + s = append(s, '0') + } + for _, r := range bufs { + s = append(s, r) + } + } else { + s = append(s, b) + } + } + } + offset += 1 + l + return string(s), offset, nil +} + +// Helpers for dealing with escaped bytes +func isDigit(b byte) bool { return b >= '0' && b <= '9' } + +func dddToByte(s []byte) byte { + return byte((s[0]-'0')*100 + (s[1]-'0')*10 + (s[2] - '0')) +} + +// Helper function for packing and unpacking +func intToBytes(i *big.Int, length int) []byte { + buf := i.Bytes() + if len(buf) < length { + b := make([]byte, length) + copy(b[length-len(buf):], buf) + return b + } + return buf +} + +// PackRR packs a resource record rr into msg[off:]. +// See PackDomainName for documentation about the compression. +func PackRR(rr RR, msg []byte, off int, compression map[string]int, compress bool) (off1 int, err error) { + if rr == nil { + return len(msg), &Error{err: "nil rr"} + } + + off1, err = rr.pack(msg, off, compression, compress) + if err != nil { + return len(msg), err + } + // TODO(miek): Not sure if this is needed? If removed we can remove rawmsg.go as well. + if rawSetRdlength(msg, off, off1) { + return off1, nil + } + return off, ErrRdata +} + +// UnpackRR unpacks msg[off:] into an RR. +func UnpackRR(msg []byte, off int) (rr RR, off1 int, err error) { + h, off, msg, err := unpackHeader(msg, off) + if err != nil { + return nil, len(msg), err + } + end := off + int(h.Rdlength) + + if fn, known := typeToUnpack[h.Rrtype]; !known { + rr, off, err = unpackRFC3597(h, msg, off) + } else { + rr, off, err = fn(h, msg, off) + } + if off != end { + return &h, end, &Error{err: "bad rdlength"} + } + return rr, off, err +} + +// unpackRRslice unpacks msg[off:] into an []RR. +// If we cannot unpack the whole array, then it will return nil +func unpackRRslice(l int, msg []byte, off int) (dst1 []RR, off1 int, err error) { + var r RR + // Optimistically make dst be the length that was sent + dst := make([]RR, 0, l) + for i := 0; i < l; i++ { + off1 := off + r, off, err = UnpackRR(msg, off) + if err != nil { + off = len(msg) + break + } + // If offset does not increase anymore, l is a lie + if off1 == off { + l = i + break + } + dst = append(dst, r) + } + if err != nil && off == len(msg) { + dst = nil + } + return dst, off, err +} + +// Convert a MsgHdr to a string, with dig-like headers: +// +//;; opcode: QUERY, status: NOERROR, id: 48404 +// +//;; flags: qr aa rd ra; +func (h *MsgHdr) String() string { + if h == nil { + return " MsgHdr" + } + + s := ";; opcode: " + OpcodeToString[h.Opcode] + s += ", status: " + RcodeToString[h.Rcode] + s += ", id: " + strconv.Itoa(int(h.Id)) + "\n" + + s += ";; flags:" + if h.Response { + s += " qr" + } + if h.Authoritative { + s += " aa" + } + if h.Truncated { + s += " tc" + } + if h.RecursionDesired { + s += " rd" + } + if h.RecursionAvailable { + s += " ra" + } + if h.Zero { // Hmm + s += " z" + } + if h.AuthenticatedData { + s += " ad" + } + if h.CheckingDisabled { + s += " cd" + } + + s += ";" + return s +} + +// Pack packs a Msg: it is converted to to wire format. +// If the dns.Compress is true the message will be in compressed wire format. +func (dns *Msg) Pack() (msg []byte, err error) { + return dns.PackBuffer(nil) +} + +// PackBuffer packs a Msg, using the given buffer buf. If buf is too small +// a new buffer is allocated. +func (dns *Msg) PackBuffer(buf []byte) (msg []byte, err error) { + // We use a similar function in tsig.go's stripTsig. + var ( + dh Header + compression map[string]int + ) + + if dns.Compress { + compression = make(map[string]int) // Compression pointer mappings + } + + if dns.Rcode < 0 || dns.Rcode > 0xFFF { + return nil, ErrRcode + } + if dns.Rcode > 0xF { + // Regular RCODE field is 4 bits + opt := dns.IsEdns0() + if opt == nil { + return nil, ErrExtendedRcode + } + opt.SetExtendedRcode(uint8(dns.Rcode >> 4)) + dns.Rcode &= 0xF + } + + // Convert convenient Msg into wire-like Header. + dh.Id = dns.Id + dh.Bits = uint16(dns.Opcode)<<11 | uint16(dns.Rcode) + if dns.Response { + dh.Bits |= _QR + } + if dns.Authoritative { + dh.Bits |= _AA + } + if dns.Truncated { + dh.Bits |= _TC + } + if dns.RecursionDesired { + dh.Bits |= _RD + } + if dns.RecursionAvailable { + dh.Bits |= _RA + } + if dns.Zero { + dh.Bits |= _Z + } + if dns.AuthenticatedData { + dh.Bits |= _AD + } + if dns.CheckingDisabled { + dh.Bits |= _CD + } + + // Prepare variable sized arrays. + question := dns.Question + answer := dns.Answer + ns := dns.Ns + extra := dns.Extra + + dh.Qdcount = uint16(len(question)) + dh.Ancount = uint16(len(answer)) + dh.Nscount = uint16(len(ns)) + dh.Arcount = uint16(len(extra)) + + // We need the uncompressed length here, because we first pack it and then compress it. + msg = buf + uncompressedLen := compressedLen(dns, false) + if packLen := uncompressedLen + 1; len(msg) < packLen { + msg = make([]byte, packLen) + } + + // Pack it in: header and then the pieces. + off := 0 + off, err = dh.pack(msg, off, compression, dns.Compress) + if err != nil { + return nil, err + } + for i := 0; i < len(question); i++ { + off, err = question[i].pack(msg, off, compression, dns.Compress) + if err != nil { + return nil, err + } + } + for i := 0; i < len(answer); i++ { + off, err = PackRR(answer[i], msg, off, compression, dns.Compress) + if err != nil { + return nil, err + } + } + for i := 0; i < len(ns); i++ { + off, err = PackRR(ns[i], msg, off, compression, dns.Compress) + if err != nil { + return nil, err + } + } + for i := 0; i < len(extra); i++ { + off, err = PackRR(extra[i], msg, off, compression, dns.Compress) + if err != nil { + return nil, err + } + } + return msg[:off], nil +} + +// Unpack unpacks a binary message to a Msg structure. +func (dns *Msg) Unpack(msg []byte) (err error) { + var ( + dh Header + off int + ) + if dh, off, err = unpackMsgHdr(msg, off); err != nil { + return err + } + + dns.Id = dh.Id + dns.Response = (dh.Bits & _QR) != 0 + dns.Opcode = int(dh.Bits>>11) & 0xF + dns.Authoritative = (dh.Bits & _AA) != 0 + dns.Truncated = (dh.Bits & _TC) != 0 + dns.RecursionDesired = (dh.Bits & _RD) != 0 + dns.RecursionAvailable = (dh.Bits & _RA) != 0 + dns.Zero = (dh.Bits & _Z) != 0 + dns.AuthenticatedData = (dh.Bits & _AD) != 0 + dns.CheckingDisabled = (dh.Bits & _CD) != 0 + dns.Rcode = int(dh.Bits & 0xF) + + if off == len(msg) { + return ErrTruncated + } + + // Optimistically use the count given to us in the header + dns.Question = make([]Question, 0, int(dh.Qdcount)) + + for i := 0; i < int(dh.Qdcount); i++ { + off1 := off + var q Question + q, off, err = unpackQuestion(msg, off) + if err != nil { + // Even if Truncated is set, we only will set ErrTruncated if we + // actually got the questions + return err + } + if off1 == off { // Offset does not increase anymore, dh.Qdcount is a lie! + dh.Qdcount = uint16(i) + break + } + dns.Question = append(dns.Question, q) + } + + dns.Answer, off, err = unpackRRslice(int(dh.Ancount), msg, off) + // The header counts might have been wrong so we need to update it + dh.Ancount = uint16(len(dns.Answer)) + if err == nil { + dns.Ns, off, err = unpackRRslice(int(dh.Nscount), msg, off) + } + // The header counts might have been wrong so we need to update it + dh.Nscount = uint16(len(dns.Ns)) + if err == nil { + dns.Extra, off, err = unpackRRslice(int(dh.Arcount), msg, off) + } + // The header counts might have been wrong so we need to update it + dh.Arcount = uint16(len(dns.Extra)) + + if off != len(msg) { + // TODO(miek) make this an error? + // use PackOpt to let people tell how detailed the error reporting should be? + // println("dns: extra bytes in dns packet", off, "<", len(msg)) + } else if dns.Truncated { + // Whether we ran into a an error or not, we want to return that it + // was truncated + err = ErrTruncated + } + return err +} + +// Convert a complete message to a string with dig-like output. +func (dns *Msg) String() string { + if dns == nil { + return " MsgHdr" + } + s := dns.MsgHdr.String() + " " + s += "QUERY: " + strconv.Itoa(len(dns.Question)) + ", " + s += "ANSWER: " + strconv.Itoa(len(dns.Answer)) + ", " + s += "AUTHORITY: " + strconv.Itoa(len(dns.Ns)) + ", " + s += "ADDITIONAL: " + strconv.Itoa(len(dns.Extra)) + "\n" + if len(dns.Question) > 0 { + s += "\n;; QUESTION SECTION:\n" + for i := 0; i < len(dns.Question); i++ { + s += dns.Question[i].String() + "\n" + } + } + if len(dns.Answer) > 0 { + s += "\n;; ANSWER SECTION:\n" + for i := 0; i < len(dns.Answer); i++ { + if dns.Answer[i] != nil { + s += dns.Answer[i].String() + "\n" + } + } + } + if len(dns.Ns) > 0 { + s += "\n;; AUTHORITY SECTION:\n" + for i := 0; i < len(dns.Ns); i++ { + if dns.Ns[i] != nil { + s += dns.Ns[i].String() + "\n" + } + } + } + if len(dns.Extra) > 0 { + s += "\n;; ADDITIONAL SECTION:\n" + for i := 0; i < len(dns.Extra); i++ { + if dns.Extra[i] != nil { + s += dns.Extra[i].String() + "\n" + } + } + } + return s +} + +// Len returns the message length when in (un)compressed wire format. +// If dns.Compress is true compression it is taken into account. Len() +// is provided to be a faster way to get the size of the resulting packet, +// than packing it, measuring the size and discarding the buffer. +func (dns *Msg) Len() int { return compressedLen(dns, dns.Compress) } + +// compressedLen returns the message length when in compressed wire format +// when compress is true, otherwise the uncompressed length is returned. +func compressedLen(dns *Msg, compress bool) int { + // We always return one more than needed. + l := 12 // Message header is always 12 bytes + compression := map[string]int{} + + for i := 0; i < len(dns.Question); i++ { + l += dns.Question[i].len() + if compress { + compressionLenHelper(compression, dns.Question[i].Name) + } + } + for i := 0; i < len(dns.Answer); i++ { + if dns.Answer[i] == nil { + continue + } + l += dns.Answer[i].len() + if compress { + k, ok := compressionLenSearch(compression, dns.Answer[i].Header().Name) + if ok { + l += 1 - k + } + compressionLenHelper(compression, dns.Answer[i].Header().Name) + k, ok = compressionLenSearchType(compression, dns.Answer[i]) + if ok { + l += 1 - k + } + compressionLenHelperType(compression, dns.Answer[i]) + } + } + for i := 0; i < len(dns.Ns); i++ { + if dns.Ns[i] == nil { + continue + } + l += dns.Ns[i].len() + if compress { + k, ok := compressionLenSearch(compression, dns.Ns[i].Header().Name) + if ok { + l += 1 - k + } + compressionLenHelper(compression, dns.Ns[i].Header().Name) + k, ok = compressionLenSearchType(compression, dns.Ns[i]) + if ok { + l += 1 - k + } + compressionLenHelperType(compression, dns.Ns[i]) + } + } + for i := 0; i < len(dns.Extra); i++ { + if dns.Extra[i] == nil { + continue + } + l += dns.Extra[i].len() + if compress { + k, ok := compressionLenSearch(compression, dns.Extra[i].Header().Name) + if ok { + l += 1 - k + } + compressionLenHelper(compression, dns.Extra[i].Header().Name) + k, ok = compressionLenSearchType(compression, dns.Extra[i]) + if ok { + l += 1 - k + } + compressionLenHelperType(compression, dns.Extra[i]) + } + } + return l +} + +// Put the parts of the name in the compression map. +func compressionLenHelper(c map[string]int, s string) { + pref := "" + lbs := Split(s) + for j := len(lbs) - 1; j >= 0; j-- { + pref = s[lbs[j]:] + if _, ok := c[pref]; !ok { + c[pref] = len(pref) + } + } +} + +// Look for each part in the compression map and returns its length, +// keep on searching so we get the longest match. +func compressionLenSearch(c map[string]int, s string) (int, bool) { + off := 0 + end := false + if s == "" { // don't bork on bogus data + return 0, false + } + for { + if _, ok := c[s[off:]]; ok { + return len(s[off:]), true + } + if end { + break + } + off, end = NextLabel(s, off) + } + return 0, false +} + +// Copy returns a new RR which is a deep-copy of r. +func Copy(r RR) RR { r1 := r.copy(); return r1 } + +// Len returns the length (in octets) of the uncompressed RR in wire format. +func Len(r RR) int { return r.len() } + +// Copy returns a new *Msg which is a deep-copy of dns. +func (dns *Msg) Copy() *Msg { return dns.CopyTo(new(Msg)) } + +// CopyTo copies the contents to the provided message using a deep-copy and returns the copy. +func (dns *Msg) CopyTo(r1 *Msg) *Msg { + r1.MsgHdr = dns.MsgHdr + r1.Compress = dns.Compress + + if len(dns.Question) > 0 { + r1.Question = make([]Question, len(dns.Question)) + copy(r1.Question, dns.Question) // TODO(miek): Question is an immutable value, ok to do a shallow-copy + } + + rrArr := make([]RR, len(dns.Answer)+len(dns.Ns)+len(dns.Extra)) + var rri int + + if len(dns.Answer) > 0 { + rrbegin := rri + for i := 0; i < len(dns.Answer); i++ { + rrArr[rri] = dns.Answer[i].copy() + rri++ + } + r1.Answer = rrArr[rrbegin:rri:rri] + } + + if len(dns.Ns) > 0 { + rrbegin := rri + for i := 0; i < len(dns.Ns); i++ { + rrArr[rri] = dns.Ns[i].copy() + rri++ + } + r1.Ns = rrArr[rrbegin:rri:rri] + } + + if len(dns.Extra) > 0 { + rrbegin := rri + for i := 0; i < len(dns.Extra); i++ { + rrArr[rri] = dns.Extra[i].copy() + rri++ + } + r1.Extra = rrArr[rrbegin:rri:rri] + } + + return r1 +} + +func (q *Question) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := PackDomainName(q.Name, msg, off, compression, compress) + if err != nil { + return off, err + } + off, err = packUint16(q.Qtype, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(q.Qclass, msg, off) + if err != nil { + return off, err + } + return off, nil +} + +func unpackQuestion(msg []byte, off int) (Question, int, error) { + var ( + q Question + err error + ) + q.Name, off, err = UnpackDomainName(msg, off) + if err != nil { + return q, off, err + } + if off == len(msg) { + return q, off, nil + } + q.Qtype, off, err = unpackUint16(msg, off) + if err != nil { + return q, off, err + } + if off == len(msg) { + return q, off, nil + } + q.Qclass, off, err = unpackUint16(msg, off) + if off == len(msg) { + return q, off, nil + } + return q, off, err +} + +func (dh *Header) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := packUint16(dh.Id, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(dh.Bits, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(dh.Qdcount, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(dh.Ancount, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(dh.Nscount, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(dh.Arcount, msg, off) + return off, err +} + +func unpackMsgHdr(msg []byte, off int) (Header, int, error) { + var ( + dh Header + err error + ) + dh.Id, off, err = unpackUint16(msg, off) + if err != nil { + return dh, off, err + } + dh.Bits, off, err = unpackUint16(msg, off) + if err != nil { + return dh, off, err + } + dh.Qdcount, off, err = unpackUint16(msg, off) + if err != nil { + return dh, off, err + } + dh.Ancount, off, err = unpackUint16(msg, off) + if err != nil { + return dh, off, err + } + dh.Nscount, off, err = unpackUint16(msg, off) + if err != nil { + return dh, off, err + } + dh.Arcount, off, err = unpackUint16(msg, off) + return dh, off, err +} diff --git a/vendor/github.com/miekg/dns/msg_generate.go b/vendor/github.com/miekg/dns/msg_generate.go new file mode 100644 index 00000000..4d9f81d4 --- /dev/null +++ b/vendor/github.com/miekg/dns/msg_generate.go @@ -0,0 +1,349 @@ +//+build ignore + +// msg_generate.go is meant to run with go generate. It will use +// go/{importer,types} to track down all the RR struct types. Then for each type +// it will generate pack/unpack methods based on the struct tags. The generated source is +// written to zmsg.go, and is meant to be checked into git. +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/importer" + "go/types" + "log" + "os" + "strings" +) + +var packageHdr = ` +// *** DO NOT MODIFY *** +// AUTOGENERATED BY go generate from msg_generate.go + +package dns + +` + +// getTypeStruct will take a type and the package scope, and return the +// (innermost) struct if the type is considered a RR type (currently defined as +// those structs beginning with a RR_Header, could be redefined as implementing +// the RR interface). The bool return value indicates if embedded structs were +// resolved. +func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { + st, ok := t.Underlying().(*types.Struct) + if !ok { + return nil, false + } + if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { + return st, false + } + if st.Field(0).Anonymous() { + st, _ := getTypeStruct(st.Field(0).Type(), scope) + return st, true + } + return nil, false +} + +func main() { + // Import and type-check the package + pkg, err := importer.Default().Import("github.com/miekg/dns") + fatalIfErr(err) + scope := pkg.Scope() + + // Collect actual types (*X) + var namedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + if st, _ := getTypeStruct(o.Type(), scope); st == nil { + continue + } + if name == "PrivateRR" { + continue + } + + // Check if corresponding TypeX exists + if scope.Lookup("Type"+o.Name()) == nil && o.Name() != "RFC3597" { + log.Fatalf("Constant Type%s does not exist.", o.Name()) + } + + namedTypes = append(namedTypes, o.Name()) + } + + b := &bytes.Buffer{} + b.WriteString(packageHdr) + + fmt.Fprint(b, "// pack*() functions\n\n") + for _, name := range namedTypes { + o := scope.Lookup(name) + st, _ := getTypeStruct(o.Type(), scope) + + fmt.Fprintf(b, "func (rr *%s) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) {\n", name) + fmt.Fprint(b, `off, err := rr.Hdr.pack(msg, off, compression, compress) +if err != nil { + return off, err +} +headerEnd := off +`) + for i := 1; i < st.NumFields(); i++ { + o := func(s string) { + fmt.Fprintf(b, s, st.Field(i).Name()) + fmt.Fprint(b, `if err != nil { +return off, err +} +`) + } + + if _, ok := st.Field(i).Type().(*types.Slice); ok { + switch st.Tag(i) { + case `dns:"-"`: // ignored + case `dns:"txt"`: + o("off, err = packStringTxt(rr.%s, msg, off)\n") + case `dns:"opt"`: + o("off, err = packDataOpt(rr.%s, msg, off)\n") + case `dns:"nsec"`: + o("off, err = packDataNsec(rr.%s, msg, off)\n") + case `dns:"domain-name"`: + o("off, err = packDataDomainNames(rr.%s, msg, off, compression, compress)\n") + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + continue + } + + switch { + case st.Tag(i) == `dns:"-"`: // ignored + case st.Tag(i) == `dns:"cdomain-name"`: + o("off, err = PackDomainName(rr.%s, msg, off, compression, compress)\n") + case st.Tag(i) == `dns:"domain-name"`: + o("off, err = PackDomainName(rr.%s, msg, off, compression, false)\n") + case st.Tag(i) == `dns:"a"`: + o("off, err = packDataA(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"aaaa"`: + o("off, err = packDataAAAA(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"uint48"`: + o("off, err = packUint48(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"txt"`: + o("off, err = packString(rr.%s, msg, off)\n") + + case strings.HasPrefix(st.Tag(i), `dns:"size-base32`): // size-base32 can be packed just like base32 + fallthrough + case st.Tag(i) == `dns:"base32"`: + o("off, err = packStringBase32(rr.%s, msg, off)\n") + + case strings.HasPrefix(st.Tag(i), `dns:"size-base64`): // size-base64 can be packed just like base64 + fallthrough + case st.Tag(i) == `dns:"base64"`: + o("off, err = packStringBase64(rr.%s, msg, off)\n") + + case strings.HasPrefix(st.Tag(i), `dns:"size-hex:SaltLength`): + // directly write instead of using o() so we get the error check in the correct place + field := st.Field(i).Name() + fmt.Fprintf(b, `// Only pack salt if value is not "-", i.e. empty +if rr.%s != "-" { + off, err = packStringHex(rr.%s, msg, off) + if err != nil { + return off, err + } +} +`, field, field) + continue + case strings.HasPrefix(st.Tag(i), `dns:"size-hex`): // size-hex can be packed just like hex + fallthrough + case st.Tag(i) == `dns:"hex"`: + o("off, err = packStringHex(rr.%s, msg, off)\n") + + case st.Tag(i) == `dns:"octet"`: + o("off, err = packStringOctet(rr.%s, msg, off)\n") + case st.Tag(i) == "": + switch st.Field(i).Type().(*types.Basic).Kind() { + case types.Uint8: + o("off, err = packUint8(rr.%s, msg, off)\n") + case types.Uint16: + o("off, err = packUint16(rr.%s, msg, off)\n") + case types.Uint32: + o("off, err = packUint32(rr.%s, msg, off)\n") + case types.Uint64: + o("off, err = packUint64(rr.%s, msg, off)\n") + case types.String: + o("off, err = packString(rr.%s, msg, off)\n") + default: + log.Fatalln(name, st.Field(i).Name()) + } + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + } + // We have packed everything, only now we know the rdlength of this RR + fmt.Fprintln(b, "rr.Header().Rdlength = uint16(off-headerEnd)") + fmt.Fprintln(b, "return off, nil }\n") + } + + fmt.Fprint(b, "// unpack*() functions\n\n") + for _, name := range namedTypes { + o := scope.Lookup(name) + st, _ := getTypeStruct(o.Type(), scope) + + fmt.Fprintf(b, "func unpack%s(h RR_Header, msg []byte, off int) (RR, int, error) {\n", name) + fmt.Fprintf(b, "rr := new(%s)\n", name) + fmt.Fprint(b, "rr.Hdr = h\n") + fmt.Fprint(b, `if noRdata(h) { +return rr, off, nil + } +var err error +rdStart := off +_ = rdStart + +`) + for i := 1; i < st.NumFields(); i++ { + o := func(s string) { + fmt.Fprintf(b, s, st.Field(i).Name()) + fmt.Fprint(b, `if err != nil { +return rr, off, err +} +`) + } + + // size-* are special, because they reference a struct member we should use for the length. + if strings.HasPrefix(st.Tag(i), `dns:"size-`) { + structMember := structMember(st.Tag(i)) + structTag := structTag(st.Tag(i)) + switch structTag { + case "hex": + fmt.Fprintf(b, "rr.%s, off, err = unpackStringHex(msg, off, off + int(rr.%s))\n", st.Field(i).Name(), structMember) + case "base32": + fmt.Fprintf(b, "rr.%s, off, err = unpackStringBase32(msg, off, off + int(rr.%s))\n", st.Field(i).Name(), structMember) + case "base64": + fmt.Fprintf(b, "rr.%s, off, err = unpackStringBase64(msg, off, off + int(rr.%s))\n", st.Field(i).Name(), structMember) + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + fmt.Fprint(b, `if err != nil { +return rr, off, err +} +`) + continue + } + + if _, ok := st.Field(i).Type().(*types.Slice); ok { + switch st.Tag(i) { + case `dns:"-"`: // ignored + case `dns:"txt"`: + o("rr.%s, off, err = unpackStringTxt(msg, off)\n") + case `dns:"opt"`: + o("rr.%s, off, err = unpackDataOpt(msg, off)\n") + case `dns:"nsec"`: + o("rr.%s, off, err = unpackDataNsec(msg, off)\n") + case `dns:"domain-name"`: + o("rr.%s, off, err = unpackDataDomainNames(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + continue + } + + switch st.Tag(i) { + case `dns:"-"`: // ignored + case `dns:"cdomain-name"`: + fallthrough + case `dns:"domain-name"`: + o("rr.%s, off, err = UnpackDomainName(msg, off)\n") + case `dns:"a"`: + o("rr.%s, off, err = unpackDataA(msg, off)\n") + case `dns:"aaaa"`: + o("rr.%s, off, err = unpackDataAAAA(msg, off)\n") + case `dns:"uint48"`: + o("rr.%s, off, err = unpackUint48(msg, off)\n") + case `dns:"txt"`: + o("rr.%s, off, err = unpackString(msg, off)\n") + case `dns:"base32"`: + o("rr.%s, off, err = unpackStringBase32(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"base64"`: + o("rr.%s, off, err = unpackStringBase64(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"hex"`: + o("rr.%s, off, err = unpackStringHex(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"octet"`: + o("rr.%s, off, err = unpackStringOctet(msg, off)\n") + case "": + switch st.Field(i).Type().(*types.Basic).Kind() { + case types.Uint8: + o("rr.%s, off, err = unpackUint8(msg, off)\n") + case types.Uint16: + o("rr.%s, off, err = unpackUint16(msg, off)\n") + case types.Uint32: + o("rr.%s, off, err = unpackUint32(msg, off)\n") + case types.Uint64: + o("rr.%s, off, err = unpackUint64(msg, off)\n") + case types.String: + o("rr.%s, off, err = unpackString(msg, off)\n") + default: + log.Fatalln(name, st.Field(i).Name()) + } + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + // If we've hit len(msg) we return without error. + if i < st.NumFields()-1 { + fmt.Fprintf(b, `if off == len(msg) { +return rr, off, nil + } +`) + } + } + fmt.Fprintf(b, "return rr, off, err }\n\n") + } + // Generate typeToUnpack map + fmt.Fprintln(b, "var typeToUnpack = map[uint16]func(RR_Header, []byte, int) (RR, int, error){") + for _, name := range namedTypes { + if name == "RFC3597" { + continue + } + fmt.Fprintf(b, "Type%s: unpack%s,\n", name, name) + } + fmt.Fprintln(b, "}\n") + + // gofmt + res, err := format.Source(b.Bytes()) + if err != nil { + b.WriteTo(os.Stderr) + log.Fatal(err) + } + + // write result + f, err := os.Create("zmsg.go") + fatalIfErr(err) + defer f.Close() + f.Write(res) +} + +// structMember will take a tag like dns:"size-base32:SaltLength" and return the last part of this string. +func structMember(s string) string { + fields := strings.Split(s, ":") + if len(fields) == 0 { + return "" + } + f := fields[len(fields)-1] + // f should have a closing " + if len(f) > 1 { + return f[:len(f)-1] + } + return f +} + +// structTag will take a tag like dns:"size-base32:SaltLength" and return base32. +func structTag(s string) string { + fields := strings.Split(s, ":") + if len(fields) < 2 { + return "" + } + return fields[1][len("\"size-"):] +} + +func fatalIfErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/vendor/github.com/miekg/dns/msg_helpers.go b/vendor/github.com/miekg/dns/msg_helpers.go new file mode 100644 index 00000000..615274ab --- /dev/null +++ b/vendor/github.com/miekg/dns/msg_helpers.go @@ -0,0 +1,633 @@ +package dns + +import ( + "encoding/base32" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "net" + "strconv" +) + +// helper functions called from the generated zmsg.go + +// These function are named after the tag to help pack/unpack, if there is no tag it is the name +// of the type they pack/unpack (string, int, etc). We prefix all with unpackData or packData, so packDataA or +// packDataDomainName. + +func unpackDataA(msg []byte, off int) (net.IP, int, error) { + if off+net.IPv4len > len(msg) { + return nil, len(msg), &Error{err: "overflow unpacking a"} + } + a := append(make(net.IP, 0, net.IPv4len), msg[off:off+net.IPv4len]...) + off += net.IPv4len + return a, off, nil +} + +func packDataA(a net.IP, msg []byte, off int) (int, error) { + // It must be a slice of 4, even if it is 16, we encode only the first 4 + if off+net.IPv4len > len(msg) { + return len(msg), &Error{err: "overflow packing a"} + } + switch len(a) { + case net.IPv4len, net.IPv6len: + copy(msg[off:], a.To4()) + off += net.IPv4len + case 0: + // Allowed, for dynamic updates. + default: + return len(msg), &Error{err: "overflow packing a"} + } + return off, nil +} + +func unpackDataAAAA(msg []byte, off int) (net.IP, int, error) { + if off+net.IPv6len > len(msg) { + return nil, len(msg), &Error{err: "overflow unpacking aaaa"} + } + aaaa := append(make(net.IP, 0, net.IPv6len), msg[off:off+net.IPv6len]...) + off += net.IPv6len + return aaaa, off, nil +} + +func packDataAAAA(aaaa net.IP, msg []byte, off int) (int, error) { + if off+net.IPv6len > len(msg) { + return len(msg), &Error{err: "overflow packing aaaa"} + } + + switch len(aaaa) { + case net.IPv6len: + copy(msg[off:], aaaa) + off += net.IPv6len + case 0: + // Allowed, dynamic updates. + default: + return len(msg), &Error{err: "overflow packing aaaa"} + } + return off, nil +} + +// unpackHeader unpacks an RR header, returning the offset to the end of the header and a +// re-sliced msg according to the expected length of the RR. +func unpackHeader(msg []byte, off int) (rr RR_Header, off1 int, truncmsg []byte, err error) { + hdr := RR_Header{} + if off == len(msg) { + return hdr, off, msg, nil + } + + hdr.Name, off, err = UnpackDomainName(msg, off) + if err != nil { + return hdr, len(msg), msg, err + } + hdr.Rrtype, off, err = unpackUint16(msg, off) + if err != nil { + return hdr, len(msg), msg, err + } + hdr.Class, off, err = unpackUint16(msg, off) + if err != nil { + return hdr, len(msg), msg, err + } + hdr.Ttl, off, err = unpackUint32(msg, off) + if err != nil { + return hdr, len(msg), msg, err + } + hdr.Rdlength, off, err = unpackUint16(msg, off) + if err != nil { + return hdr, len(msg), msg, err + } + msg, err = truncateMsgFromRdlength(msg, off, hdr.Rdlength) + return hdr, off, msg, nil +} + +// pack packs an RR header, returning the offset to the end of the header. +// See PackDomainName for documentation about the compression. +func (hdr RR_Header) pack(msg []byte, off int, compression map[string]int, compress bool) (off1 int, err error) { + if off == len(msg) { + return off, nil + } + + off, err = PackDomainName(hdr.Name, msg, off, compression, compress) + if err != nil { + return len(msg), err + } + off, err = packUint16(hdr.Rrtype, msg, off) + if err != nil { + return len(msg), err + } + off, err = packUint16(hdr.Class, msg, off) + if err != nil { + return len(msg), err + } + off, err = packUint32(hdr.Ttl, msg, off) + if err != nil { + return len(msg), err + } + off, err = packUint16(hdr.Rdlength, msg, off) + if err != nil { + return len(msg), err + } + return off, nil +} + +// helper helper functions. + +// truncateMsgFromRdLength truncates msg to match the expected length of the RR. +// Returns an error if msg is smaller than the expected size. +func truncateMsgFromRdlength(msg []byte, off int, rdlength uint16) (truncmsg []byte, err error) { + lenrd := off + int(rdlength) + if lenrd > len(msg) { + return msg, &Error{err: "overflowing header size"} + } + return msg[:lenrd], nil +} + +func fromBase32(s []byte) (buf []byte, err error) { + for i, b := range s { + if b >= 'a' && b <= 'z' { + s[i] = b - 32 + } + } + buflen := base32.HexEncoding.DecodedLen(len(s)) + buf = make([]byte, buflen) + n, err := base32.HexEncoding.Decode(buf, s) + buf = buf[:n] + return +} + +func toBase32(b []byte) string { return base32.HexEncoding.EncodeToString(b) } + +func fromBase64(s []byte) (buf []byte, err error) { + buflen := base64.StdEncoding.DecodedLen(len(s)) + buf = make([]byte, buflen) + n, err := base64.StdEncoding.Decode(buf, s) + buf = buf[:n] + return +} + +func toBase64(b []byte) string { return base64.StdEncoding.EncodeToString(b) } + +// dynamicUpdate returns true if the Rdlength is zero. +func noRdata(h RR_Header) bool { return h.Rdlength == 0 } + +func unpackUint8(msg []byte, off int) (i uint8, off1 int, err error) { + if off+1 > len(msg) { + return 0, len(msg), &Error{err: "overflow unpacking uint8"} + } + return uint8(msg[off]), off + 1, nil +} + +func packUint8(i uint8, msg []byte, off int) (off1 int, err error) { + if off+1 > len(msg) { + return len(msg), &Error{err: "overflow packing uint8"} + } + msg[off] = byte(i) + return off + 1, nil +} + +func unpackUint16(msg []byte, off int) (i uint16, off1 int, err error) { + if off+2 > len(msg) { + return 0, len(msg), &Error{err: "overflow unpacking uint16"} + } + return binary.BigEndian.Uint16(msg[off:]), off + 2, nil +} + +func packUint16(i uint16, msg []byte, off int) (off1 int, err error) { + if off+2 > len(msg) { + return len(msg), &Error{err: "overflow packing uint16"} + } + binary.BigEndian.PutUint16(msg[off:], i) + return off + 2, nil +} + +func unpackUint32(msg []byte, off int) (i uint32, off1 int, err error) { + if off+4 > len(msg) { + return 0, len(msg), &Error{err: "overflow unpacking uint32"} + } + return binary.BigEndian.Uint32(msg[off:]), off + 4, nil +} + +func packUint32(i uint32, msg []byte, off int) (off1 int, err error) { + if off+4 > len(msg) { + return len(msg), &Error{err: "overflow packing uint32"} + } + binary.BigEndian.PutUint32(msg[off:], i) + return off + 4, nil +} + +func unpackUint48(msg []byte, off int) (i uint64, off1 int, err error) { + if off+6 > len(msg) { + return 0, len(msg), &Error{err: "overflow unpacking uint64 as uint48"} + } + // Used in TSIG where the last 48 bits are occupied, so for now, assume a uint48 (6 bytes) + i = (uint64(uint64(msg[off])<<40 | uint64(msg[off+1])<<32 | uint64(msg[off+2])<<24 | uint64(msg[off+3])<<16 | + uint64(msg[off+4])<<8 | uint64(msg[off+5]))) + off += 6 + return i, off, nil +} + +func packUint48(i uint64, msg []byte, off int) (off1 int, err error) { + if off+6 > len(msg) { + return len(msg), &Error{err: "overflow packing uint64 as uint48"} + } + msg[off] = byte(i >> 40) + msg[off+1] = byte(i >> 32) + msg[off+2] = byte(i >> 24) + msg[off+3] = byte(i >> 16) + msg[off+4] = byte(i >> 8) + msg[off+5] = byte(i) + off += 6 + return off, nil +} + +func unpackUint64(msg []byte, off int) (i uint64, off1 int, err error) { + if off+8 > len(msg) { + return 0, len(msg), &Error{err: "overflow unpacking uint64"} + } + return binary.BigEndian.Uint64(msg[off:]), off + 8, nil +} + +func packUint64(i uint64, msg []byte, off int) (off1 int, err error) { + if off+8 > len(msg) { + return len(msg), &Error{err: "overflow packing uint64"} + } + binary.BigEndian.PutUint64(msg[off:], i) + off += 8 + return off, nil +} + +func unpackString(msg []byte, off int) (string, int, error) { + if off+1 > len(msg) { + return "", off, &Error{err: "overflow unpacking txt"} + } + l := int(msg[off]) + if off+l+1 > len(msg) { + return "", off, &Error{err: "overflow unpacking txt"} + } + s := make([]byte, 0, l) + for _, b := range msg[off+1 : off+1+l] { + switch b { + case '"', '\\': + s = append(s, '\\', b) + default: + if b < 32 || b > 127 { // unprintable + var buf [3]byte + bufs := strconv.AppendInt(buf[:0], int64(b), 10) + s = append(s, '\\') + for i := 0; i < 3-len(bufs); i++ { + s = append(s, '0') + } + for _, r := range bufs { + s = append(s, r) + } + } else { + s = append(s, b) + } + } + } + off += 1 + l + return string(s), off, nil +} + +func packString(s string, msg []byte, off int) (int, error) { + txtTmp := make([]byte, 256*4+1) + off, err := packTxtString(s, msg, off, txtTmp) + if err != nil { + return len(msg), err + } + return off, nil +} + +func unpackStringBase32(msg []byte, off, end int) (string, int, error) { + if end > len(msg) { + return "", len(msg), &Error{err: "overflow unpacking base32"} + } + s := toBase32(msg[off:end]) + return s, end, nil +} + +func packStringBase32(s string, msg []byte, off int) (int, error) { + b32, err := fromBase32([]byte(s)) + if err != nil { + return len(msg), err + } + if off+len(b32) > len(msg) { + return len(msg), &Error{err: "overflow packing base32"} + } + copy(msg[off:off+len(b32)], b32) + off += len(b32) + return off, nil +} + +func unpackStringBase64(msg []byte, off, end int) (string, int, error) { + // Rest of the RR is base64 encoded value, so we don't need an explicit length + // to be set. Thus far all RR's that have base64 encoded fields have those as their + // last one. What we do need is the end of the RR! + if end > len(msg) { + return "", len(msg), &Error{err: "overflow unpacking base64"} + } + s := toBase64(msg[off:end]) + return s, end, nil +} + +func packStringBase64(s string, msg []byte, off int) (int, error) { + b64, err := fromBase64([]byte(s)) + if err != nil { + return len(msg), err + } + if off+len(b64) > len(msg) { + return len(msg), &Error{err: "overflow packing base64"} + } + copy(msg[off:off+len(b64)], b64) + off += len(b64) + return off, nil +} + +func unpackStringHex(msg []byte, off, end int) (string, int, error) { + // Rest of the RR is hex encoded value, so we don't need an explicit length + // to be set. NSEC and TSIG have hex fields with a length field. + // What we do need is the end of the RR! + if end > len(msg) { + return "", len(msg), &Error{err: "overflow unpacking hex"} + } + + s := hex.EncodeToString(msg[off:end]) + return s, end, nil +} + +func packStringHex(s string, msg []byte, off int) (int, error) { + h, err := hex.DecodeString(s) + if err != nil { + return len(msg), err + } + if off+(len(h)) > len(msg) { + return len(msg), &Error{err: "overflow packing hex"} + } + copy(msg[off:off+len(h)], h) + off += len(h) + return off, nil +} + +func unpackStringTxt(msg []byte, off int) ([]string, int, error) { + txt, off, err := unpackTxt(msg, off) + if err != nil { + return nil, len(msg), err + } + return txt, off, nil +} + +func packStringTxt(s []string, msg []byte, off int) (int, error) { + txtTmp := make([]byte, 256*4+1) // If the whole string consists out of \DDD we need this many. + off, err := packTxt(s, msg, off, txtTmp) + if err != nil { + return len(msg), err + } + return off, nil +} + +func unpackDataOpt(msg []byte, off int) ([]EDNS0, int, error) { + var edns []EDNS0 +Option: + code := uint16(0) + if off+4 > len(msg) { + return nil, len(msg), &Error{err: "overflow unpacking opt"} + } + code = binary.BigEndian.Uint16(msg[off:]) + off += 2 + optlen := binary.BigEndian.Uint16(msg[off:]) + off += 2 + if off+int(optlen) > len(msg) { + return nil, len(msg), &Error{err: "overflow unpacking opt"} + } + switch code { + case EDNS0NSID: + e := new(EDNS0_NSID) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + case EDNS0SUBNET, EDNS0SUBNETDRAFT: + e := new(EDNS0_SUBNET) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + if code == EDNS0SUBNETDRAFT { + e.DraftOption = true + } + case EDNS0COOKIE: + e := new(EDNS0_COOKIE) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + case EDNS0UL: + e := new(EDNS0_UL) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + case EDNS0LLQ: + e := new(EDNS0_LLQ) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + case EDNS0DAU: + e := new(EDNS0_DAU) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + case EDNS0DHU: + e := new(EDNS0_DHU) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + case EDNS0N3U: + e := new(EDNS0_N3U) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + default: + e := new(EDNS0_LOCAL) + e.Code = code + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) + } + + if off < len(msg) { + goto Option + } + + return edns, off, nil +} + +func packDataOpt(options []EDNS0, msg []byte, off int) (int, error) { + for _, el := range options { + b, err := el.pack() + if err != nil || off+3 > len(msg) { + return len(msg), &Error{err: "overflow packing opt"} + } + binary.BigEndian.PutUint16(msg[off:], el.Option()) // Option code + binary.BigEndian.PutUint16(msg[off+2:], uint16(len(b))) // Length + off += 4 + if off+len(b) > len(msg) { + copy(msg[off:], b) + off = len(msg) + continue + } + // Actual data + copy(msg[off:off+len(b)], b) + off += len(b) + } + return off, nil +} + +func unpackStringOctet(msg []byte, off int) (string, int, error) { + s := string(msg[off:]) + return s, len(msg), nil +} + +func packStringOctet(s string, msg []byte, off int) (int, error) { + txtTmp := make([]byte, 256*4+1) + off, err := packOctetString(s, msg, off, txtTmp) + if err != nil { + return len(msg), err + } + return off, nil +} + +func unpackDataNsec(msg []byte, off int) ([]uint16, int, error) { + var nsec []uint16 + length, window, lastwindow := 0, 0, -1 + for off < len(msg) { + if off+2 > len(msg) { + return nsec, len(msg), &Error{err: "overflow unpacking nsecx"} + } + window = int(msg[off]) + length = int(msg[off+1]) + off += 2 + if window <= lastwindow { + // RFC 4034: Blocks are present in the NSEC RR RDATA in + // increasing numerical order. + return nsec, len(msg), &Error{err: "out of order NSEC block"} + } + if length == 0 { + // RFC 4034: Blocks with no types present MUST NOT be included. + return nsec, len(msg), &Error{err: "empty NSEC block"} + } + if length > 32 { + return nsec, len(msg), &Error{err: "NSEC block too long"} + } + if off+length > len(msg) { + return nsec, len(msg), &Error{err: "overflowing NSEC block"} + } + + // Walk the bytes in the window and extract the type bits + for j := 0; j < length; j++ { + b := msg[off+j] + // Check the bits one by one, and set the type + if b&0x80 == 0x80 { + nsec = append(nsec, uint16(window*256+j*8+0)) + } + if b&0x40 == 0x40 { + nsec = append(nsec, uint16(window*256+j*8+1)) + } + if b&0x20 == 0x20 { + nsec = append(nsec, uint16(window*256+j*8+2)) + } + if b&0x10 == 0x10 { + nsec = append(nsec, uint16(window*256+j*8+3)) + } + if b&0x8 == 0x8 { + nsec = append(nsec, uint16(window*256+j*8+4)) + } + if b&0x4 == 0x4 { + nsec = append(nsec, uint16(window*256+j*8+5)) + } + if b&0x2 == 0x2 { + nsec = append(nsec, uint16(window*256+j*8+6)) + } + if b&0x1 == 0x1 { + nsec = append(nsec, uint16(window*256+j*8+7)) + } + } + off += length + lastwindow = window + } + return nsec, off, nil +} + +func packDataNsec(bitmap []uint16, msg []byte, off int) (int, error) { + if len(bitmap) == 0 { + return off, nil + } + var lastwindow, lastlength uint16 + for j := 0; j < len(bitmap); j++ { + t := bitmap[j] + window := t / 256 + length := (t-window*256)/8 + 1 + if window > lastwindow && lastlength != 0 { // New window, jump to the new offset + off += int(lastlength) + 2 + lastlength = 0 + } + if window < lastwindow || length < lastlength { + return len(msg), &Error{err: "nsec bits out of order"} + } + if off+2+int(length) > len(msg) { + return len(msg), &Error{err: "overflow packing nsec"} + } + // Setting the window # + msg[off] = byte(window) + // Setting the octets length + msg[off+1] = byte(length) + // Setting the bit value for the type in the right octet + msg[off+1+int(length)] |= byte(1 << (7 - (t % 8))) + lastwindow, lastlength = window, length + } + off += int(lastlength) + 2 + return off, nil +} + +func unpackDataDomainNames(msg []byte, off, end int) ([]string, int, error) { + var ( + servers []string + s string + err error + ) + if end > len(msg) { + return nil, len(msg), &Error{err: "overflow unpacking domain names"} + } + for off < end { + s, off, err = UnpackDomainName(msg, off) + if err != nil { + return servers, len(msg), err + } + servers = append(servers, s) + } + return servers, off, nil +} + +func packDataDomainNames(names []string, msg []byte, off int, compression map[string]int, compress bool) (int, error) { + var err error + for j := 0; j < len(names); j++ { + off, err = PackDomainName(names[j], msg, off, compression, false && compress) + if err != nil { + return len(msg), err + } + } + return off, nil +} diff --git a/vendor/github.com/miekg/dns/nsecx.go b/vendor/github.com/miekg/dns/nsecx.go new file mode 100644 index 00000000..9b908c44 --- /dev/null +++ b/vendor/github.com/miekg/dns/nsecx.go @@ -0,0 +1,106 @@ +package dns + +import ( + "crypto/sha1" + "hash" + "strings" +) + +type saltWireFmt struct { + Salt string `dns:"size-hex"` +} + +// HashName hashes a string (label) according to RFC 5155. It returns the hashed string in uppercase. +func HashName(label string, ha uint8, iter uint16, salt string) string { + saltwire := new(saltWireFmt) + saltwire.Salt = salt + wire := make([]byte, DefaultMsgSize) + n, err := packSaltWire(saltwire, wire) + if err != nil { + return "" + } + wire = wire[:n] + name := make([]byte, 255) + off, err := PackDomainName(strings.ToLower(label), name, 0, nil, false) + if err != nil { + return "" + } + name = name[:off] + var s hash.Hash + switch ha { + case SHA1: + s = sha1.New() + default: + return "" + } + + // k = 0 + s.Write(name) + s.Write(wire) + nsec3 := s.Sum(nil) + // k > 0 + for k := uint16(0); k < iter; k++ { + s.Reset() + s.Write(nsec3) + s.Write(wire) + nsec3 = s.Sum(nsec3[:0]) + } + return toBase32(nsec3) +} + +// Cover returns true if a name is covered by the NSEC3 record +func (rr *NSEC3) Cover(name string) bool { + nameHash := HashName(name, rr.Hash, rr.Iterations, rr.Salt) + owner := strings.ToUpper(rr.Hdr.Name) + labelIndices := Split(owner) + if len(labelIndices) < 2 { + return false + } + ownerHash := owner[:labelIndices[1]-1] + ownerZone := owner[labelIndices[1]:] + if !IsSubDomain(ownerZone, strings.ToUpper(name)) { // name is outside owner zone + return false + } + + nextHash := rr.NextDomain + if ownerHash == nextHash { // empty interval + return false + } + if ownerHash > nextHash { // end of zone + if nameHash > ownerHash { // covered since there is nothing after ownerHash + return true + } + return nameHash < nextHash // if nameHash is before beginning of zone it is covered + } + if nameHash < ownerHash { // nameHash is before ownerHash, not covered + return false + } + return nameHash < nextHash // if nameHash is before nextHash is it covered (between ownerHash and nextHash) +} + +// Match returns true if a name matches the NSEC3 record +func (rr *NSEC3) Match(name string) bool { + nameHash := HashName(name, rr.Hash, rr.Iterations, rr.Salt) + owner := strings.ToUpper(rr.Hdr.Name) + labelIndices := Split(owner) + if len(labelIndices) < 2 { + return false + } + ownerHash := owner[:labelIndices[1]-1] + ownerZone := owner[labelIndices[1]:] + if !IsSubDomain(ownerZone, strings.ToUpper(name)) { // name is outside owner zone + return false + } + if ownerHash == nameHash { + return true + } + return false +} + +func packSaltWire(sw *saltWireFmt, msg []byte) (int, error) { + off, err := packStringHex(sw.Salt, msg, 0) + if err != nil { + return off, err + } + return off, nil +} diff --git a/vendor/github.com/miekg/dns/privaterr.go b/vendor/github.com/miekg/dns/privaterr.go new file mode 100644 index 00000000..6b08e6e9 --- /dev/null +++ b/vendor/github.com/miekg/dns/privaterr.go @@ -0,0 +1,149 @@ +package dns + +import ( + "fmt" + "strings" +) + +// PrivateRdata is an interface used for implementing "Private Use" RR types, see +// RFC 6895. This allows one to experiment with new RR types, without requesting an +// official type code. Also see dns.PrivateHandle and dns.PrivateHandleRemove. +type PrivateRdata interface { + // String returns the text presentaton of the Rdata of the Private RR. + String() string + // Parse parses the Rdata of the private RR. + Parse([]string) error + // Pack is used when packing a private RR into a buffer. + Pack([]byte) (int, error) + // Unpack is used when unpacking a private RR from a buffer. + // TODO(miek): diff. signature than Pack, see edns0.go for instance. + Unpack([]byte) (int, error) + // Copy copies the Rdata. + Copy(PrivateRdata) error + // Len returns the length in octets of the Rdata. + Len() int +} + +// PrivateRR represents an RR that uses a PrivateRdata user-defined type. +// It mocks normal RRs and implements dns.RR interface. +type PrivateRR struct { + Hdr RR_Header + Data PrivateRdata +} + +func mkPrivateRR(rrtype uint16) *PrivateRR { + // Panics if RR is not an instance of PrivateRR. + rrfunc, ok := TypeToRR[rrtype] + if !ok { + panic(fmt.Sprintf("dns: invalid operation with Private RR type %d", rrtype)) + } + + anyrr := rrfunc() + switch rr := anyrr.(type) { + case *PrivateRR: + return rr + } + panic(fmt.Sprintf("dns: RR is not a PrivateRR, TypeToRR[%d] generator returned %T", rrtype, anyrr)) +} + +// Header return the RR header of r. +func (r *PrivateRR) Header() *RR_Header { return &r.Hdr } + +func (r *PrivateRR) String() string { return r.Hdr.String() + r.Data.String() } + +// Private len and copy parts to satisfy RR interface. +func (r *PrivateRR) len() int { return r.Hdr.len() + r.Data.Len() } +func (r *PrivateRR) copy() RR { + // make new RR like this: + rr := mkPrivateRR(r.Hdr.Rrtype) + newh := r.Hdr.copyHeader() + rr.Hdr = *newh + + err := r.Data.Copy(rr.Data) + if err != nil { + panic("dns: got value that could not be used to copy Private rdata") + } + return rr +} +func (r *PrivateRR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := r.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + n, err := r.Data.Pack(msg[off:]) + if err != nil { + return len(msg), err + } + off += n + r.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +// PrivateHandle registers a private resource record type. It requires +// string and numeric representation of private RR type and generator function as argument. +func PrivateHandle(rtypestr string, rtype uint16, generator func() PrivateRdata) { + rtypestr = strings.ToUpper(rtypestr) + + TypeToRR[rtype] = func() RR { return &PrivateRR{RR_Header{}, generator()} } + TypeToString[rtype] = rtypestr + StringToType[rtypestr] = rtype + + typeToUnpack[rtype] = func(h RR_Header, msg []byte, off int) (RR, int, error) { + if noRdata(h) { + return &h, off, nil + } + var err error + + rr := mkPrivateRR(h.Rrtype) + rr.Hdr = h + + off1, err := rr.Data.Unpack(msg[off:]) + off += off1 + if err != nil { + return rr, off, err + } + return rr, off, err + } + + setPrivateRR := func(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := mkPrivateRR(h.Rrtype) + rr.Hdr = h + + var l lex + text := make([]string, 0, 2) // could be 0..N elements, median is probably 1 + Fetch: + for { + // TODO(miek): we could also be returning _QUOTE, this might or might not + // be an issue (basically parsing TXT becomes hard) + switch l = <-c; l.value { + case zNewline, zEOF: + break Fetch + case zString: + text = append(text, l.token) + } + } + + err := rr.Data.Parse(text) + if err != nil { + return nil, &ParseError{f, err.Error(), l}, "" + } + + return rr, nil, "" + } + + typeToparserFunc[rtype] = parserFunc{setPrivateRR, true} +} + +// PrivateHandleRemove removes defenitions required to support private RR type. +func PrivateHandleRemove(rtype uint16) { + rtypestr, ok := TypeToString[rtype] + if ok { + delete(TypeToRR, rtype) + delete(TypeToString, rtype) + delete(typeToparserFunc, rtype) + delete(StringToType, rtypestr) + delete(typeToUnpack, rtype) + } + return +} diff --git a/vendor/github.com/miekg/dns/rawmsg.go b/vendor/github.com/miekg/dns/rawmsg.go new file mode 100644 index 00000000..6e21fba7 --- /dev/null +++ b/vendor/github.com/miekg/dns/rawmsg.go @@ -0,0 +1,49 @@ +package dns + +import "encoding/binary" + +// rawSetRdlength sets the rdlength in the header of +// the RR. The offset 'off' must be positioned at the +// start of the header of the RR, 'end' must be the +// end of the RR. +func rawSetRdlength(msg []byte, off, end int) bool { + l := len(msg) +Loop: + for { + if off+1 > l { + return false + } + c := int(msg[off]) + off++ + switch c & 0xC0 { + case 0x00: + if c == 0x00 { + // End of the domainname + break Loop + } + if off+c > l { + return false + } + off += c + + case 0xC0: + // pointer, next byte included, ends domainname + off++ + break Loop + } + } + // The domainname has been seen, we at the start of the fixed part in the header. + // Type is 2 bytes, class is 2 bytes, ttl 4 and then 2 bytes for the length. + off += 2 + 2 + 4 + if off+2 > l { + return false + } + //off+1 is the end of the header, 'end' is the end of the rr + //so 'end' - 'off+2' is the length of the rdata + rdatalen := end - (off + 2) + if rdatalen > 0xFFFF { + return false + } + binary.BigEndian.PutUint16(msg[off:], uint16(rdatalen)) + return true +} diff --git a/vendor/github.com/miekg/dns/reverse.go b/vendor/github.com/miekg/dns/reverse.go new file mode 100644 index 00000000..f6e7a47a --- /dev/null +++ b/vendor/github.com/miekg/dns/reverse.go @@ -0,0 +1,38 @@ +package dns + +// StringToType is the reverse of TypeToString, needed for string parsing. +var StringToType = reverseInt16(TypeToString) + +// StringToClass is the reverse of ClassToString, needed for string parsing. +var StringToClass = reverseInt16(ClassToString) + +// StringToOpcode is a map of opcodes to strings. +var StringToOpcode = reverseInt(OpcodeToString) + +// StringToRcode is a map of rcodes to strings. +var StringToRcode = reverseInt(RcodeToString) + +// Reverse a map +func reverseInt8(m map[uint8]string) map[string]uint8 { + n := make(map[string]uint8, len(m)) + for u, s := range m { + n[s] = u + } + return n +} + +func reverseInt16(m map[uint16]string) map[string]uint16 { + n := make(map[string]uint16, len(m)) + for u, s := range m { + n[s] = u + } + return n +} + +func reverseInt(m map[int]string) map[string]int { + n := make(map[string]int, len(m)) + for u, s := range m { + n[s] = u + } + return n +} diff --git a/vendor/github.com/miekg/dns/sanitize.go b/vendor/github.com/miekg/dns/sanitize.go new file mode 100644 index 00000000..b489f3f0 --- /dev/null +++ b/vendor/github.com/miekg/dns/sanitize.go @@ -0,0 +1,84 @@ +package dns + +// Dedup removes identical RRs from rrs. It preserves the original ordering. +// The lowest TTL of any duplicates is used in the remaining one. Dedup modifies +// rrs. +// m is used to store the RRs temporay. If it is nil a new map will be allocated. +func Dedup(rrs []RR, m map[string]RR) []RR { + if m == nil { + m = make(map[string]RR) + } + // Save the keys, so we don't have to call normalizedString twice. + keys := make([]*string, 0, len(rrs)) + + for _, r := range rrs { + key := normalizedString(r) + keys = append(keys, &key) + if _, ok := m[key]; ok { + // Shortest TTL wins. + if m[key].Header().Ttl > r.Header().Ttl { + m[key].Header().Ttl = r.Header().Ttl + } + continue + } + + m[key] = r + } + // If the length of the result map equals the amount of RRs we got, + // it means they were all different. We can then just return the original rrset. + if len(m) == len(rrs) { + return rrs + } + + j := 0 + for i, r := range rrs { + // If keys[i] lives in the map, we should copy and remove it. + if _, ok := m[*keys[i]]; ok { + delete(m, *keys[i]) + rrs[j] = r + j++ + } + + if len(m) == 0 { + break + } + } + + return rrs[:j] +} + +// normalizedString returns a normalized string from r. The TTL +// is removed and the domain name is lowercased. We go from this: +// DomainNameTTLCLASSTYPERDATA to: +// lowercasenameCLASSTYPE... +func normalizedString(r RR) string { + // A string Go DNS makes has: domainnameTTL... + b := []byte(r.String()) + + // find the first non-escaped tab, then another, so we capture where the TTL lives. + esc := false + ttlStart, ttlEnd := 0, 0 + for i := 0; i < len(b) && ttlEnd == 0; i++ { + switch { + case b[i] == '\\': + esc = !esc + case b[i] == '\t' && !esc: + if ttlStart == 0 { + ttlStart = i + continue + } + if ttlEnd == 0 { + ttlEnd = i + } + case b[i] >= 'A' && b[i] <= 'Z' && !esc: + b[i] += 32 + default: + esc = false + } + } + + // remove TTL. + copy(b[ttlStart:], b[ttlEnd:]) + cut := ttlEnd - ttlStart + return string(b[:len(b)-cut]) +} diff --git a/vendor/github.com/miekg/dns/scan.go b/vendor/github.com/miekg/dns/scan.go new file mode 100644 index 00000000..de0db32f --- /dev/null +++ b/vendor/github.com/miekg/dns/scan.go @@ -0,0 +1,981 @@ +package dns + +import ( + "io" + "log" + "os" + "strconv" + "strings" +) + +type debugging bool + +const debug debugging = false + +func (d debugging) Printf(format string, args ...interface{}) { + if d { + log.Printf(format, args...) + } +} + +const maxTok = 2048 // Largest token we can return. +const maxUint16 = 1<<16 - 1 + +// Tokinize a RFC 1035 zone file. The tokenizer will normalize it: +// * Add ownernames if they are left blank; +// * Suppress sequences of spaces; +// * Make each RR fit on one line (_NEWLINE is send as last) +// * Handle comments: ; +// * Handle braces - anywhere. +const ( + // Zonefile + zEOF = iota + zString + zBlank + zQuote + zNewline + zRrtpe + zOwner + zClass + zDirOrigin // $ORIGIN + zDirTtl // $TTL + zDirInclude // $INCLUDE + zDirGenerate // $GENERATE + + // Privatekey file + zValue + zKey + + zExpectOwnerDir // Ownername + zExpectOwnerBl // Whitespace after the ownername + zExpectAny // Expect rrtype, ttl or class + zExpectAnyNoClass // Expect rrtype or ttl + zExpectAnyNoClassBl // The whitespace after _EXPECT_ANY_NOCLASS + zExpectAnyNoTtl // Expect rrtype or class + zExpectAnyNoTtlBl // Whitespace after _EXPECT_ANY_NOTTL + zExpectRrtype // Expect rrtype + zExpectRrtypeBl // Whitespace BEFORE rrtype + zExpectRdata // The first element of the rdata + zExpectDirTtlBl // Space after directive $TTL + zExpectDirTtl // Directive $TTL + zExpectDirOriginBl // Space after directive $ORIGIN + zExpectDirOrigin // Directive $ORIGIN + zExpectDirIncludeBl // Space after directive $INCLUDE + zExpectDirInclude // Directive $INCLUDE + zExpectDirGenerate // Directive $GENERATE + zExpectDirGenerateBl // Space after directive $GENERATE +) + +// ParseError is a parsing error. It contains the parse error and the location in the io.Reader +// where the error occurred. +type ParseError struct { + file string + err string + lex lex +} + +func (e *ParseError) Error() (s string) { + if e.file != "" { + s = e.file + ": " + } + s += "dns: " + e.err + ": " + strconv.QuoteToASCII(e.lex.token) + " at line: " + + strconv.Itoa(e.lex.line) + ":" + strconv.Itoa(e.lex.column) + return +} + +type lex struct { + token string // text of the token + tokenUpper string // uppercase text of the token + length int // length of the token + err bool // when true, token text has lexer error + value uint8 // value: zString, _BLANK, etc. + line int // line in the file + column int // column in the file + torc uint16 // type or class as parsed in the lexer, we only need to look this up in the grammar + comment string // any comment text seen +} + +// Token holds the token that are returned when a zone file is parsed. +type Token struct { + // The scanned resource record when error is not nil. + RR + // When an error occurred, this has the error specifics. + Error *ParseError + // A potential comment positioned after the RR and on the same line. + Comment string +} + +// NewRR reads the RR contained in the string s. Only the first RR is +// returned. If s contains no RR, return nil with no error. The class +// defaults to IN and TTL defaults to 3600. The full zone file syntax +// like $TTL, $ORIGIN, etc. is supported. All fields of the returned +// RR are set, except RR.Header().Rdlength which is set to 0. +func NewRR(s string) (RR, error) { + if len(s) > 0 && s[len(s)-1] != '\n' { // We need a closing newline + return ReadRR(strings.NewReader(s+"\n"), "") + } + return ReadRR(strings.NewReader(s), "") +} + +// ReadRR reads the RR contained in q. +// See NewRR for more documentation. +func ReadRR(q io.Reader, filename string) (RR, error) { + r := <-parseZoneHelper(q, ".", filename, 1) + if r == nil { + return nil, nil + } + + if r.Error != nil { + return nil, r.Error + } + return r.RR, nil +} + +// ParseZone reads a RFC 1035 style zonefile from r. It returns *Tokens on the +// returned channel, which consist out the parsed RR, a potential comment or an error. +// If there is an error the RR is nil. The string file is only used +// in error reporting. The string origin is used as the initial origin, as +// if the file would start with: $ORIGIN origin . +// The directives $INCLUDE, $ORIGIN, $TTL and $GENERATE are supported. +// The channel t is closed by ParseZone when the end of r is reached. +// +// Basic usage pattern when reading from a string (z) containing the +// zone data: +// +// for x := range dns.ParseZone(strings.NewReader(z), "", "") { +// if x.Error != nil { +// // log.Println(x.Error) +// } else { +// // Do something with x.RR +// } +// } +// +// Comments specified after an RR (and on the same line!) are returned too: +// +// foo. IN A 10.0.0.1 ; this is a comment +// +// The text "; this is comment" is returned in Token.Comment. Comments inside the +// RR are discarded. Comments on a line by themselves are discarded too. +func ParseZone(r io.Reader, origin, file string) chan *Token { + return parseZoneHelper(r, origin, file, 10000) +} + +func parseZoneHelper(r io.Reader, origin, file string, chansize int) chan *Token { + t := make(chan *Token, chansize) + go parseZone(r, origin, file, t, 0) + return t +} + +func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { + defer func() { + if include == 0 { + close(t) + } + }() + s := scanInit(r) + c := make(chan lex) + // Start the lexer + go zlexer(s, c) + // 6 possible beginnings of a line, _ is a space + // 0. zRRTYPE -> all omitted until the rrtype + // 1. zOwner _ zRrtype -> class/ttl omitted + // 2. zOwner _ zString _ zRrtype -> class omitted + // 3. zOwner _ zString _ zClass _ zRrtype -> ttl/class + // 4. zOwner _ zClass _ zRrtype -> ttl omitted + // 5. zOwner _ zClass _ zString _ zRrtype -> class/ttl (reversed) + // After detecting these, we know the zRrtype so we can jump to functions + // handling the rdata for each of these types. + + if origin == "" { + origin = "." + } + origin = Fqdn(origin) + if _, ok := IsDomainName(origin); !ok { + t <- &Token{Error: &ParseError{f, "bad initial origin name", lex{}}} + return + } + + st := zExpectOwnerDir // initial state + var h RR_Header + var defttl uint32 = defaultTtl + var prevName string + for l := range c { + // Lexer spotted an error already + if l.err == true { + t <- &Token{Error: &ParseError{f, l.token, l}} + return + + } + switch st { + case zExpectOwnerDir: + // We can also expect a directive, like $TTL or $ORIGIN + h.Ttl = defttl + h.Class = ClassINET + switch l.value { + case zNewline: + st = zExpectOwnerDir + case zOwner: + h.Name = l.token + if l.token[0] == '@' { + h.Name = origin + prevName = h.Name + st = zExpectOwnerBl + break + } + if h.Name[l.length-1] != '.' { + h.Name = appendOrigin(h.Name, origin) + } + _, ok := IsDomainName(l.token) + if !ok { + t <- &Token{Error: &ParseError{f, "bad owner name", l}} + return + } + prevName = h.Name + st = zExpectOwnerBl + case zDirTtl: + st = zExpectDirTtlBl + case zDirOrigin: + st = zExpectDirOriginBl + case zDirInclude: + st = zExpectDirIncludeBl + case zDirGenerate: + st = zExpectDirGenerateBl + case zRrtpe: + h.Name = prevName + h.Rrtype = l.torc + st = zExpectRdata + case zClass: + h.Name = prevName + h.Class = l.torc + st = zExpectAnyNoClassBl + case zBlank: + // Discard, can happen when there is nothing on the + // line except the RR type + case zString: + ttl, ok := stringToTtl(l.token) + if !ok { + t <- &Token{Error: &ParseError{f, "not a TTL", l}} + return + } + h.Ttl = ttl + // Don't about the defttl, we should take the $TTL value + // defttl = ttl + st = zExpectAnyNoTtlBl + + default: + t <- &Token{Error: &ParseError{f, "syntax error at beginning", l}} + return + } + case zExpectDirIncludeBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank after $INCLUDE-directive", l}} + return + } + st = zExpectDirInclude + case zExpectDirInclude: + if l.value != zString { + t <- &Token{Error: &ParseError{f, "expecting $INCLUDE value, not this...", l}} + return + } + neworigin := origin // There may be optionally a new origin set after the filename, if not use current one + l := <-c + switch l.value { + case zBlank: + l := <-c + if l.value == zString { + if _, ok := IsDomainName(l.token); !ok || l.length == 0 || l.err { + t <- &Token{Error: &ParseError{f, "bad origin name", l}} + return + } + // a new origin is specified. + if l.token[l.length-1] != '.' { + if origin != "." { // Prevent .. endings + neworigin = l.token + "." + origin + } else { + neworigin = l.token + origin + } + } else { + neworigin = l.token + } + } + case zNewline, zEOF: + // Ok + default: + t <- &Token{Error: &ParseError{f, "garbage after $INCLUDE", l}} + return + } + // Start with the new file + r1, e1 := os.Open(l.token) + if e1 != nil { + t <- &Token{Error: &ParseError{f, "failed to open `" + l.token + "'", l}} + return + } + if include+1 > 7 { + t <- &Token{Error: &ParseError{f, "too deeply nested $INCLUDE", l}} + return + } + parseZone(r1, l.token, neworigin, t, include+1) + st = zExpectOwnerDir + case zExpectDirTtlBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank after $TTL-directive", l}} + return + } + st = zExpectDirTtl + case zExpectDirTtl: + if l.value != zString { + t <- &Token{Error: &ParseError{f, "expecting $TTL value, not this...", l}} + return + } + if e, _ := slurpRemainder(c, f); e != nil { + t <- &Token{Error: e} + return + } + ttl, ok := stringToTtl(l.token) + if !ok { + t <- &Token{Error: &ParseError{f, "expecting $TTL value, not this...", l}} + return + } + defttl = ttl + st = zExpectOwnerDir + case zExpectDirOriginBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank after $ORIGIN-directive", l}} + return + } + st = zExpectDirOrigin + case zExpectDirOrigin: + if l.value != zString { + t <- &Token{Error: &ParseError{f, "expecting $ORIGIN value, not this...", l}} + return + } + if e, _ := slurpRemainder(c, f); e != nil { + t <- &Token{Error: e} + } + if _, ok := IsDomainName(l.token); !ok { + t <- &Token{Error: &ParseError{f, "bad origin name", l}} + return + } + if l.token[l.length-1] != '.' { + if origin != "." { // Prevent .. endings + origin = l.token + "." + origin + } else { + origin = l.token + origin + } + } else { + origin = l.token + } + st = zExpectOwnerDir + case zExpectDirGenerateBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank after $GENERATE-directive", l}} + return + } + st = zExpectDirGenerate + case zExpectDirGenerate: + if l.value != zString { + t <- &Token{Error: &ParseError{f, "expecting $GENERATE value, not this...", l}} + return + } + if errMsg := generate(l, c, t, origin); errMsg != "" { + t <- &Token{Error: &ParseError{f, errMsg, l}} + return + } + st = zExpectOwnerDir + case zExpectOwnerBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank after owner", l}} + return + } + st = zExpectAny + case zExpectAny: + switch l.value { + case zRrtpe: + h.Rrtype = l.torc + st = zExpectRdata + case zClass: + h.Class = l.torc + st = zExpectAnyNoClassBl + case zString: + ttl, ok := stringToTtl(l.token) + if !ok { + t <- &Token{Error: &ParseError{f, "not a TTL", l}} + return + } + h.Ttl = ttl + // defttl = ttl // don't set the defttl here + st = zExpectAnyNoTtlBl + default: + t <- &Token{Error: &ParseError{f, "expecting RR type, TTL or class, not this...", l}} + return + } + case zExpectAnyNoClassBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank before class", l}} + return + } + st = zExpectAnyNoClass + case zExpectAnyNoTtlBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank before TTL", l}} + return + } + st = zExpectAnyNoTtl + case zExpectAnyNoTtl: + switch l.value { + case zClass: + h.Class = l.torc + st = zExpectRrtypeBl + case zRrtpe: + h.Rrtype = l.torc + st = zExpectRdata + default: + t <- &Token{Error: &ParseError{f, "expecting RR type or class, not this...", l}} + return + } + case zExpectAnyNoClass: + switch l.value { + case zString: + ttl, ok := stringToTtl(l.token) + if !ok { + t <- &Token{Error: &ParseError{f, "not a TTL", l}} + return + } + h.Ttl = ttl + // defttl = ttl // don't set the def ttl anymore + st = zExpectRrtypeBl + case zRrtpe: + h.Rrtype = l.torc + st = zExpectRdata + default: + t <- &Token{Error: &ParseError{f, "expecting RR type or TTL, not this...", l}} + return + } + case zExpectRrtypeBl: + if l.value != zBlank { + t <- &Token{Error: &ParseError{f, "no blank before RR type", l}} + return + } + st = zExpectRrtype + case zExpectRrtype: + if l.value != zRrtpe { + t <- &Token{Error: &ParseError{f, "unknown RR type", l}} + return + } + h.Rrtype = l.torc + st = zExpectRdata + case zExpectRdata: + r, e, c1 := setRR(h, c, origin, f) + if e != nil { + // If e.lex is nil than we have encounter a unknown RR type + // in that case we substitute our current lex token + if e.lex.token == "" && e.lex.value == 0 { + e.lex = l // Uh, dirty + } + t <- &Token{Error: e} + return + } + t <- &Token{RR: r, Comment: c1} + st = zExpectOwnerDir + } + } + // If we get here, we and the h.Rrtype is still zero, we haven't parsed anything, this + // is not an error, because an empty zone file is still a zone file. +} + +// zlexer scans the sourcefile and returns tokens on the channel c. +func zlexer(s *scan, c chan lex) { + var l lex + str := make([]byte, maxTok) // Should be enough for any token + stri := 0 // Offset in str (0 means empty) + com := make([]byte, maxTok) // Hold comment text + comi := 0 + quote := false + escape := false + space := false + commt := false + rrtype := false + owner := true + brace := 0 + x, err := s.tokenText() + defer close(c) + for err == nil { + l.column = s.position.Column + l.line = s.position.Line + if stri >= maxTok { + l.token = "token length insufficient for parsing" + l.err = true + debug.Printf("[%+v]", l.token) + c <- l + return + } + if comi >= maxTok { + l.token = "comment length insufficient for parsing" + l.err = true + debug.Printf("[%+v]", l.token) + c <- l + return + } + + switch x { + case ' ', '\t': + if escape { + escape = false + str[stri] = x + stri++ + break + } + if quote { + // Inside quotes this is legal + str[stri] = x + stri++ + break + } + if commt { + com[comi] = x + comi++ + break + } + if stri == 0 { + // Space directly in the beginning, handled in the grammar + } else if owner { + // If we have a string and its the first, make it an owner + l.value = zOwner + l.token = string(str[:stri]) + l.tokenUpper = strings.ToUpper(l.token) + l.length = stri + // escape $... start with a \ not a $, so this will work + switch l.tokenUpper { + case "$TTL": + l.value = zDirTtl + case "$ORIGIN": + l.value = zDirOrigin + case "$INCLUDE": + l.value = zDirInclude + case "$GENERATE": + l.value = zDirGenerate + } + debug.Printf("[7 %+v]", l.token) + c <- l + } else { + l.value = zString + l.token = string(str[:stri]) + l.tokenUpper = strings.ToUpper(l.token) + l.length = stri + if !rrtype { + if t, ok := StringToType[l.tokenUpper]; ok { + l.value = zRrtpe + l.torc = t + rrtype = true + } else { + if strings.HasPrefix(l.tokenUpper, "TYPE") { + t, ok := typeToInt(l.token) + if !ok { + l.token = "unknown RR type" + l.err = true + c <- l + return + } + l.value = zRrtpe + l.torc = t + } + } + if t, ok := StringToClass[l.tokenUpper]; ok { + l.value = zClass + l.torc = t + } else { + if strings.HasPrefix(l.tokenUpper, "CLASS") { + t, ok := classToInt(l.token) + if !ok { + l.token = "unknown class" + l.err = true + c <- l + return + } + l.value = zClass + l.torc = t + } + } + } + debug.Printf("[6 %+v]", l.token) + c <- l + } + stri = 0 + // I reverse space stuff here + if !space && !commt { + l.value = zBlank + l.token = " " + l.length = 1 + debug.Printf("[5 %+v]", l.token) + c <- l + } + owner = false + space = true + case ';': + if escape { + escape = false + str[stri] = x + stri++ + break + } + if quote { + // Inside quotes this is legal + str[stri] = x + stri++ + break + } + if stri > 0 { + l.value = zString + l.token = string(str[:stri]) + l.tokenUpper = strings.ToUpper(l.token) + l.length = stri + debug.Printf("[4 %+v]", l.token) + c <- l + stri = 0 + } + commt = true + com[comi] = ';' + comi++ + case '\r': + escape = false + if quote { + str[stri] = x + stri++ + break + } + // discard if outside of quotes + case '\n': + escape = false + // Escaped newline + if quote { + str[stri] = x + stri++ + break + } + // inside quotes this is legal + if commt { + // Reset a comment + commt = false + rrtype = false + stri = 0 + // If not in a brace this ends the comment AND the RR + if brace == 0 { + owner = true + owner = true + l.value = zNewline + l.token = "\n" + l.tokenUpper = l.token + l.length = 1 + l.comment = string(com[:comi]) + debug.Printf("[3 %+v %+v]", l.token, l.comment) + c <- l + l.comment = "" + comi = 0 + break + } + com[comi] = ' ' // convert newline to space + comi++ + break + } + + if brace == 0 { + // If there is previous text, we should output it here + if stri != 0 { + l.value = zString + l.token = string(str[:stri]) + l.tokenUpper = strings.ToUpper(l.token) + + l.length = stri + if !rrtype { + if t, ok := StringToType[l.tokenUpper]; ok { + l.value = zRrtpe + l.torc = t + rrtype = true + } + } + debug.Printf("[2 %+v]", l.token) + c <- l + } + l.value = zNewline + l.token = "\n" + l.tokenUpper = l.token + l.length = 1 + debug.Printf("[1 %+v]", l.token) + c <- l + stri = 0 + commt = false + rrtype = false + owner = true + comi = 0 + } + case '\\': + // comments do not get escaped chars, everything is copied + if commt { + com[comi] = x + comi++ + break + } + // something already escaped must be in string + if escape { + str[stri] = x + stri++ + escape = false + break + } + // something escaped outside of string gets added to string + str[stri] = x + stri++ + escape = true + case '"': + if commt { + com[comi] = x + comi++ + break + } + if escape { + str[stri] = x + stri++ + escape = false + break + } + space = false + // send previous gathered text and the quote + if stri != 0 { + l.value = zString + l.token = string(str[:stri]) + l.tokenUpper = strings.ToUpper(l.token) + l.length = stri + + debug.Printf("[%+v]", l.token) + c <- l + stri = 0 + } + + // send quote itself as separate token + l.value = zQuote + l.token = "\"" + l.tokenUpper = l.token + l.length = 1 + c <- l + quote = !quote + case '(', ')': + if commt { + com[comi] = x + comi++ + break + } + if escape { + str[stri] = x + stri++ + escape = false + break + } + if quote { + str[stri] = x + stri++ + break + } + switch x { + case ')': + brace-- + if brace < 0 { + l.token = "extra closing brace" + l.tokenUpper = l.token + l.err = true + debug.Printf("[%+v]", l.token) + c <- l + return + } + case '(': + brace++ + } + default: + escape = false + if commt { + com[comi] = x + comi++ + break + } + str[stri] = x + stri++ + space = false + } + x, err = s.tokenText() + } + if stri > 0 { + // Send remainder + l.token = string(str[:stri]) + l.tokenUpper = strings.ToUpper(l.token) + l.length = stri + l.value = zString + debug.Printf("[%+v]", l.token) + c <- l + } +} + +// Extract the class number from CLASSxx +func classToInt(token string) (uint16, bool) { + offset := 5 + if len(token) < offset+1 { + return 0, false + } + class, err := strconv.ParseUint(token[offset:], 10, 16) + if err != nil { + return 0, false + } + return uint16(class), true +} + +// Extract the rr number from TYPExxx +func typeToInt(token string) (uint16, bool) { + offset := 4 + if len(token) < offset+1 { + return 0, false + } + typ, err := strconv.ParseUint(token[offset:], 10, 16) + if err != nil { + return 0, false + } + return uint16(typ), true +} + +// Parse things like 2w, 2m, etc, Return the time in seconds. +func stringToTtl(token string) (uint32, bool) { + s := uint32(0) + i := uint32(0) + for _, c := range token { + switch c { + case 's', 'S': + s += i + i = 0 + case 'm', 'M': + s += i * 60 + i = 0 + case 'h', 'H': + s += i * 60 * 60 + i = 0 + case 'd', 'D': + s += i * 60 * 60 * 24 + i = 0 + case 'w', 'W': + s += i * 60 * 60 * 24 * 7 + i = 0 + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + i *= 10 + i += uint32(c) - '0' + default: + return 0, false + } + } + return s + i, true +} + +// Parse LOC records' [.][mM] into a +// mantissa exponent format. Token should contain the entire +// string (i.e. no spaces allowed) +func stringToCm(token string) (e, m uint8, ok bool) { + if token[len(token)-1] == 'M' || token[len(token)-1] == 'm' { + token = token[0 : len(token)-1] + } + s := strings.SplitN(token, ".", 2) + var meters, cmeters, val int + var err error + switch len(s) { + case 2: + if cmeters, err = strconv.Atoi(s[1]); err != nil { + return + } + fallthrough + case 1: + if meters, err = strconv.Atoi(s[0]); err != nil { + return + } + case 0: + // huh? + return 0, 0, false + } + ok = true + if meters > 0 { + e = 2 + val = meters + } else { + e = 0 + val = cmeters + } + for val > 10 { + e++ + val /= 10 + } + if e > 9 { + ok = false + } + m = uint8(val) + return +} + +func appendOrigin(name, origin string) string { + if origin == "." { + return name + origin + } + return name + "." + origin +} + +// LOC record helper function +func locCheckNorth(token string, latitude uint32) (uint32, bool) { + switch token { + case "n", "N": + return LOC_EQUATOR + latitude, true + case "s", "S": + return LOC_EQUATOR - latitude, true + } + return latitude, false +} + +// LOC record helper function +func locCheckEast(token string, longitude uint32) (uint32, bool) { + switch token { + case "e", "E": + return LOC_EQUATOR + longitude, true + case "w", "W": + return LOC_EQUATOR - longitude, true + } + return longitude, false +} + +// "Eat" the rest of the "line". Return potential comments +func slurpRemainder(c chan lex, f string) (*ParseError, string) { + l := <-c + com := "" + switch l.value { + case zBlank: + l = <-c + com = l.comment + if l.value != zNewline && l.value != zEOF { + return &ParseError{f, "garbage after rdata", l}, "" + } + case zNewline: + com = l.comment + case zEOF: + default: + return &ParseError{f, "garbage after rdata", l}, "" + } + return nil, com +} + +// Parse a 64 bit-like ipv6 address: "0014:4fff:ff20:ee64" +// Used for NID and L64 record. +func stringToNodeID(l lex) (uint64, *ParseError) { + if len(l.token) < 19 { + return 0, &ParseError{l.token, "bad NID/L64 NodeID/Locator64", l} + } + // There must be three colons at fixes postitions, if not its a parse error + if l.token[4] != ':' && l.token[9] != ':' && l.token[14] != ':' { + return 0, &ParseError{l.token, "bad NID/L64 NodeID/Locator64", l} + } + s := l.token[0:4] + l.token[5:9] + l.token[10:14] + l.token[15:19] + u, err := strconv.ParseUint(s, 16, 64) + if err != nil { + return 0, &ParseError{l.token, "bad NID/L64 NodeID/Locator64", l} + } + return u, nil +} diff --git a/vendor/github.com/miekg/dns/scan_rr.go b/vendor/github.com/miekg/dns/scan_rr.go new file mode 100644 index 00000000..b8b18fd7 --- /dev/null +++ b/vendor/github.com/miekg/dns/scan_rr.go @@ -0,0 +1,2184 @@ +package dns + +import ( + "encoding/base64" + "net" + "strconv" + "strings" +) + +type parserFunc struct { + // Func defines the function that parses the tokens and returns the RR + // or an error. The last string contains any comments in the line as + // they returned by the lexer as well. + Func func(h RR_Header, c chan lex, origin string, file string) (RR, *ParseError, string) + // Signals if the RR ending is of variable length, like TXT or records + // that have Hexadecimal or Base64 as their last element in the Rdata. Records + // that have a fixed ending or for instance A, AAAA, SOA and etc. + Variable bool +} + +// Parse the rdata of each rrtype. +// All data from the channel c is either zString or zBlank. +// After the rdata there may come a zBlank and then a zNewline +// or immediately a zNewline. If this is not the case we flag +// an *ParseError: garbage after rdata. +func setRR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + parserfunc, ok := typeToparserFunc[h.Rrtype] + if ok { + r, e, cm := parserfunc.Func(h, c, o, f) + if parserfunc.Variable { + return r, e, cm + } + if e != nil { + return nil, e, "" + } + e, cm = slurpRemainder(c, f) + if e != nil { + return nil, e, "" + } + return r, nil, cm + } + // RFC3957 RR (Unknown RR handling) + return setRFC3597(h, c, o, f) +} + +// A remainder of the rdata with embedded spaces, return the parsed string (sans the spaces) +// or an error +func endingToString(c chan lex, errstr, f string) (string, *ParseError, string) { + s := "" + l := <-c // zString + for l.value != zNewline && l.value != zEOF { + if l.err { + return s, &ParseError{f, errstr, l}, "" + } + switch l.value { + case zString: + s += l.token + case zBlank: // Ok + default: + return "", &ParseError{f, errstr, l}, "" + } + l = <-c + } + return s, nil, l.comment +} + +// A remainder of the rdata with embedded spaces, split on unquoted whitespace +// and return the parsed string slice or an error +func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, string) { + // Get the remaining data until we see a zNewline + l := <-c + if l.err { + return nil, &ParseError{f, errstr, l}, "" + } + + // Build the slice + s := make([]string, 0) + quote := false + empty := false + for l.value != zNewline && l.value != zEOF { + if l.err { + return nil, &ParseError{f, errstr, l}, "" + } + switch l.value { + case zString: + empty = false + if len(l.token) > 255 { + // split up tokens that are larger than 255 into 255-chunks + sx := []string{} + p, i := 0, 255 + for { + if i <= len(l.token) { + sx = append(sx, l.token[p:i]) + } else { + sx = append(sx, l.token[p:]) + break + + } + p, i = p+255, i+255 + } + s = append(s, sx...) + break + } + + s = append(s, l.token) + case zBlank: + if quote { + // zBlank can only be seen in between txt parts. + return nil, &ParseError{f, errstr, l}, "" + } + case zQuote: + if empty && quote { + s = append(s, "") + } + quote = !quote + empty = true + default: + return nil, &ParseError{f, errstr, l}, "" + } + l = <-c + } + if quote { + return nil, &ParseError{f, errstr, l}, "" + } + return s, nil, l.comment +} + +func setA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(A) + rr.Hdr = h + + l := <-c + if l.length == 0 { // Dynamic updates. + return rr, nil, "" + } + rr.A = net.ParseIP(l.token) + if rr.A == nil || l.err { + return nil, &ParseError{f, "bad A A", l}, "" + } + return rr, nil, "" +} + +func setAAAA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(AAAA) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + rr.AAAA = net.ParseIP(l.token) + if rr.AAAA == nil || l.err { + return nil, &ParseError{f, "bad AAAA AAAA", l}, "" + } + return rr, nil, "" +} + +func setNS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NS) + rr.Hdr = h + + l := <-c + rr.Ns = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Ns = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad NS Ns", l}, "" + } + if rr.Ns[l.length-1] != '.' { + rr.Ns = appendOrigin(rr.Ns, o) + } + return rr, nil, "" +} + +func setPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(PTR) + rr.Hdr = h + + l := <-c + rr.Ptr = l.token + if l.length == 0 { // dynamic update rr. + return rr, nil, "" + } + if l.token == "@" { + rr.Ptr = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad PTR Ptr", l}, "" + } + if rr.Ptr[l.length-1] != '.' { + rr.Ptr = appendOrigin(rr.Ptr, o) + } + return rr, nil, "" +} + +func setNSAPPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NSAPPTR) + rr.Hdr = h + + l := <-c + rr.Ptr = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Ptr = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad NSAP-PTR Ptr", l}, "" + } + if rr.Ptr[l.length-1] != '.' { + rr.Ptr = appendOrigin(rr.Ptr, o) + } + return rr, nil, "" +} + +func setRP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(RP) + rr.Hdr = h + + l := <-c + rr.Mbox = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Mbox = o + } else { + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad RP Mbox", l}, "" + } + if rr.Mbox[l.length-1] != '.' { + rr.Mbox = appendOrigin(rr.Mbox, o) + } + } + <-c // zBlank + l = <-c + rr.Txt = l.token + if l.token == "@" { + rr.Txt = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad RP Txt", l}, "" + } + if rr.Txt[l.length-1] != '.' { + rr.Txt = appendOrigin(rr.Txt, o) + } + return rr, nil, "" +} + +func setMR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(MR) + rr.Hdr = h + + l := <-c + rr.Mr = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Mr = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MR Mr", l}, "" + } + if rr.Mr[l.length-1] != '.' { + rr.Mr = appendOrigin(rr.Mr, o) + } + return rr, nil, "" +} + +func setMB(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(MB) + rr.Hdr = h + + l := <-c + rr.Mb = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Mb = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MB Mb", l}, "" + } + if rr.Mb[l.length-1] != '.' { + rr.Mb = appendOrigin(rr.Mb, o) + } + return rr, nil, "" +} + +func setMG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(MG) + rr.Hdr = h + + l := <-c + rr.Mg = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Mg = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MG Mg", l}, "" + } + if rr.Mg[l.length-1] != '.' { + rr.Mg = appendOrigin(rr.Mg, o) + } + return rr, nil, "" +} + +func setHINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(HINFO) + rr.Hdr = h + + chunks, e, c1 := endingToTxtSlice(c, "bad HINFO Fields", f) + if e != nil { + return nil, e, c1 + } + + if ln := len(chunks); ln == 0 { + return rr, nil, "" + } else if ln == 1 { + // Can we split it? + if out := strings.Fields(chunks[0]); len(out) > 1 { + chunks = out + } else { + chunks = append(chunks, "") + } + } + + rr.Cpu = chunks[0] + rr.Os = strings.Join(chunks[1:], " ") + + return rr, nil, "" +} + +func setMINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(MINFO) + rr.Hdr = h + + l := <-c + rr.Rmail = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Rmail = o + } else { + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MINFO Rmail", l}, "" + } + if rr.Rmail[l.length-1] != '.' { + rr.Rmail = appendOrigin(rr.Rmail, o) + } + } + <-c // zBlank + l = <-c + rr.Email = l.token + if l.token == "@" { + rr.Email = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MINFO Email", l}, "" + } + if rr.Email[l.length-1] != '.' { + rr.Email = appendOrigin(rr.Email, o) + } + return rr, nil, "" +} + +func setMF(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(MF) + rr.Hdr = h + + l := <-c + rr.Mf = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Mf = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MF Mf", l}, "" + } + if rr.Mf[l.length-1] != '.' { + rr.Mf = appendOrigin(rr.Mf, o) + } + return rr, nil, "" +} + +func setMD(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(MD) + rr.Hdr = h + + l := <-c + rr.Md = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Md = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MD Md", l}, "" + } + if rr.Md[l.length-1] != '.' { + rr.Md = appendOrigin(rr.Md, o) + } + return rr, nil, "" +} + +func setMX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(MX) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad MX Pref", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Mx = l.token + if l.token == "@" { + rr.Mx = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad MX Mx", l}, "" + } + if rr.Mx[l.length-1] != '.' { + rr.Mx = appendOrigin(rr.Mx, o) + } + return rr, nil, "" +} + +func setRT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(RT) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil { + return nil, &ParseError{f, "bad RT Preference", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Host = l.token + if l.token == "@" { + rr.Host = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad RT Host", l}, "" + } + if rr.Host[l.length-1] != '.' { + rr.Host = appendOrigin(rr.Host, o) + } + return rr, nil, "" +} + +func setAFSDB(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(AFSDB) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad AFSDB Subtype", l}, "" + } + rr.Subtype = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Hostname = l.token + if l.token == "@" { + rr.Hostname = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad AFSDB Hostname", l}, "" + } + if rr.Hostname[l.length-1] != '.' { + rr.Hostname = appendOrigin(rr.Hostname, o) + } + return rr, nil, "" +} + +func setX25(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(X25) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + if l.err { + return nil, &ParseError{f, "bad X25 PSDNAddress", l}, "" + } + rr.PSDNAddress = l.token + return rr, nil, "" +} + +func setKX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(KX) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad KX Pref", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Exchanger = l.token + if l.token == "@" { + rr.Exchanger = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad KX Exchanger", l}, "" + } + if rr.Exchanger[l.length-1] != '.' { + rr.Exchanger = appendOrigin(rr.Exchanger, o) + } + return rr, nil, "" +} + +func setCNAME(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(CNAME) + rr.Hdr = h + + l := <-c + rr.Target = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Target = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad CNAME Target", l}, "" + } + if rr.Target[l.length-1] != '.' { + rr.Target = appendOrigin(rr.Target, o) + } + return rr, nil, "" +} + +func setDNAME(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(DNAME) + rr.Hdr = h + + l := <-c + rr.Target = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Target = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad CNAME Target", l}, "" + } + if rr.Target[l.length-1] != '.' { + rr.Target = appendOrigin(rr.Target, o) + } + return rr, nil, "" +} + +func setSOA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(SOA) + rr.Hdr = h + + l := <-c + rr.Ns = l.token + if l.length == 0 { + return rr, nil, "" + } + <-c // zBlank + if l.token == "@" { + rr.Ns = o + } else { + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad SOA Ns", l}, "" + } + if rr.Ns[l.length-1] != '.' { + rr.Ns = appendOrigin(rr.Ns, o) + } + } + + l = <-c + rr.Mbox = l.token + if l.token == "@" { + rr.Mbox = o + } else { + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad SOA Mbox", l}, "" + } + if rr.Mbox[l.length-1] != '.' { + rr.Mbox = appendOrigin(rr.Mbox, o) + } + } + <-c // zBlank + + var ( + v uint32 + ok bool + ) + for i := 0; i < 5; i++ { + l = <-c + if l.err { + return nil, &ParseError{f, "bad SOA zone parameter", l}, "" + } + if j, e := strconv.ParseUint(l.token, 10, 32); e != nil { + if i == 0 { + // Serial should be a number + return nil, &ParseError{f, "bad SOA zone parameter", l}, "" + } + if v, ok = stringToTtl(l.token); !ok { + return nil, &ParseError{f, "bad SOA zone parameter", l}, "" + + } + } else { + v = uint32(j) + } + switch i { + case 0: + rr.Serial = v + <-c // zBlank + case 1: + rr.Refresh = v + <-c // zBlank + case 2: + rr.Retry = v + <-c // zBlank + case 3: + rr.Expire = v + <-c // zBlank + case 4: + rr.Minttl = v + } + } + return rr, nil, "" +} + +func setSRV(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(SRV) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad SRV Priority", l}, "" + } + rr.Priority = uint16(i) + <-c // zBlank + l = <-c // zString + i, e = strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad SRV Weight", l}, "" + } + rr.Weight = uint16(i) + <-c // zBlank + l = <-c // zString + i, e = strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad SRV Port", l}, "" + } + rr.Port = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Target = l.token + if l.token == "@" { + rr.Target = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad SRV Target", l}, "" + } + if rr.Target[l.length-1] != '.' { + rr.Target = appendOrigin(rr.Target, o) + } + return rr, nil, "" +} + +func setNAPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NAPTR) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad NAPTR Order", l}, "" + } + rr.Order = uint16(i) + <-c // zBlank + l = <-c // zString + i, e = strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad NAPTR Preference", l}, "" + } + rr.Preference = uint16(i) + // Flags + <-c // zBlank + l = <-c // _QUOTE + if l.value != zQuote { + return nil, &ParseError{f, "bad NAPTR Flags", l}, "" + } + l = <-c // Either String or Quote + if l.value == zString { + rr.Flags = l.token + l = <-c // _QUOTE + if l.value != zQuote { + return nil, &ParseError{f, "bad NAPTR Flags", l}, "" + } + } else if l.value == zQuote { + rr.Flags = "" + } else { + return nil, &ParseError{f, "bad NAPTR Flags", l}, "" + } + + // Service + <-c // zBlank + l = <-c // _QUOTE + if l.value != zQuote { + return nil, &ParseError{f, "bad NAPTR Service", l}, "" + } + l = <-c // Either String or Quote + if l.value == zString { + rr.Service = l.token + l = <-c // _QUOTE + if l.value != zQuote { + return nil, &ParseError{f, "bad NAPTR Service", l}, "" + } + } else if l.value == zQuote { + rr.Service = "" + } else { + return nil, &ParseError{f, "bad NAPTR Service", l}, "" + } + + // Regexp + <-c // zBlank + l = <-c // _QUOTE + if l.value != zQuote { + return nil, &ParseError{f, "bad NAPTR Regexp", l}, "" + } + l = <-c // Either String or Quote + if l.value == zString { + rr.Regexp = l.token + l = <-c // _QUOTE + if l.value != zQuote { + return nil, &ParseError{f, "bad NAPTR Regexp", l}, "" + } + } else if l.value == zQuote { + rr.Regexp = "" + } else { + return nil, &ParseError{f, "bad NAPTR Regexp", l}, "" + } + // After quote no space?? + <-c // zBlank + l = <-c // zString + rr.Replacement = l.token + if l.token == "@" { + rr.Replacement = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad NAPTR Replacement", l}, "" + } + if rr.Replacement[l.length-1] != '.' { + rr.Replacement = appendOrigin(rr.Replacement, o) + } + return rr, nil, "" +} + +func setTALINK(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(TALINK) + rr.Hdr = h + + l := <-c + rr.PreviousName = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.PreviousName = o + } else { + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad TALINK PreviousName", l}, "" + } + if rr.PreviousName[l.length-1] != '.' { + rr.PreviousName = appendOrigin(rr.PreviousName, o) + } + } + <-c // zBlank + l = <-c + rr.NextName = l.token + if l.token == "@" { + rr.NextName = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad TALINK NextName", l}, "" + } + if rr.NextName[l.length-1] != '.' { + rr.NextName = appendOrigin(rr.NextName, o) + } + return rr, nil, "" +} + +func setLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(LOC) + rr.Hdr = h + // Non zero defaults for LOC record, see RFC 1876, Section 3. + rr.HorizPre = 165 // 10000 + rr.VertPre = 162 // 10 + rr.Size = 18 // 1 + ok := false + // North + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 32) + if e != nil || l.err { + return nil, &ParseError{f, "bad LOC Latitude", l}, "" + } + rr.Latitude = 1000 * 60 * 60 * uint32(i) + + <-c // zBlank + // Either number, 'N' or 'S' + l = <-c + if rr.Latitude, ok = locCheckNorth(l.token, rr.Latitude); ok { + goto East + } + i, e = strconv.ParseUint(l.token, 10, 32) + if e != nil || l.err { + return nil, &ParseError{f, "bad LOC Latitude minutes", l}, "" + } + rr.Latitude += 1000 * 60 * uint32(i) + + <-c // zBlank + l = <-c + if i, e := strconv.ParseFloat(l.token, 32); e != nil || l.err { + return nil, &ParseError{f, "bad LOC Latitude seconds", l}, "" + } else { + rr.Latitude += uint32(1000 * i) + } + <-c // zBlank + // Either number, 'N' or 'S' + l = <-c + if rr.Latitude, ok = locCheckNorth(l.token, rr.Latitude); ok { + goto East + } + // If still alive, flag an error + return nil, &ParseError{f, "bad LOC Latitude North/South", l}, "" + +East: + // East + <-c // zBlank + l = <-c + if i, e := strconv.ParseUint(l.token, 10, 32); e != nil || l.err { + return nil, &ParseError{f, "bad LOC Longitude", l}, "" + } else { + rr.Longitude = 1000 * 60 * 60 * uint32(i) + } + <-c // zBlank + // Either number, 'E' or 'W' + l = <-c + if rr.Longitude, ok = locCheckEast(l.token, rr.Longitude); ok { + goto Altitude + } + if i, e := strconv.ParseUint(l.token, 10, 32); e != nil || l.err { + return nil, &ParseError{f, "bad LOC Longitude minutes", l}, "" + } else { + rr.Longitude += 1000 * 60 * uint32(i) + } + <-c // zBlank + l = <-c + if i, e := strconv.ParseFloat(l.token, 32); e != nil || l.err { + return nil, &ParseError{f, "bad LOC Longitude seconds", l}, "" + } else { + rr.Longitude += uint32(1000 * i) + } + <-c // zBlank + // Either number, 'E' or 'W' + l = <-c + if rr.Longitude, ok = locCheckEast(l.token, rr.Longitude); ok { + goto Altitude + } + // If still alive, flag an error + return nil, &ParseError{f, "bad LOC Longitude East/West", l}, "" + +Altitude: + <-c // zBlank + l = <-c + if l.length == 0 || l.err { + return nil, &ParseError{f, "bad LOC Altitude", l}, "" + } + if l.token[len(l.token)-1] == 'M' || l.token[len(l.token)-1] == 'm' { + l.token = l.token[0 : len(l.token)-1] + } + if i, e := strconv.ParseFloat(l.token, 32); e != nil { + return nil, &ParseError{f, "bad LOC Altitude", l}, "" + } else { + rr.Altitude = uint32(i*100.0 + 10000000.0 + 0.5) + } + + // And now optionally the other values + l = <-c + count := 0 + for l.value != zNewline && l.value != zEOF { + switch l.value { + case zString: + switch count { + case 0: // Size + e, m, ok := stringToCm(l.token) + if !ok { + return nil, &ParseError{f, "bad LOC Size", l}, "" + } + rr.Size = (e & 0x0f) | (m << 4 & 0xf0) + case 1: // HorizPre + e, m, ok := stringToCm(l.token) + if !ok { + return nil, &ParseError{f, "bad LOC HorizPre", l}, "" + } + rr.HorizPre = (e & 0x0f) | (m << 4 & 0xf0) + case 2: // VertPre + e, m, ok := stringToCm(l.token) + if !ok { + return nil, &ParseError{f, "bad LOC VertPre", l}, "" + } + rr.VertPre = (e & 0x0f) | (m << 4 & 0xf0) + } + count++ + case zBlank: + // Ok + default: + return nil, &ParseError{f, "bad LOC Size, HorizPre or VertPre", l}, "" + } + l = <-c + } + return rr, nil, "" +} + +func setHIP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(HIP) + rr.Hdr = h + + // HitLength is not represented + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad HIP PublicKeyAlgorithm", l}, "" + } + rr.PublicKeyAlgorithm = uint8(i) + <-c // zBlank + l = <-c // zString + if l.length == 0 || l.err { + return nil, &ParseError{f, "bad HIP Hit", l}, "" + } + rr.Hit = l.token // This can not contain spaces, see RFC 5205 Section 6. + rr.HitLength = uint8(len(rr.Hit)) / 2 + + <-c // zBlank + l = <-c // zString + if l.length == 0 || l.err { + return nil, &ParseError{f, "bad HIP PublicKey", l}, "" + } + rr.PublicKey = l.token // This cannot contain spaces + rr.PublicKeyLength = uint16(base64.StdEncoding.DecodedLen(len(rr.PublicKey))) + + // RendezvousServers (if any) + l = <-c + var xs []string + for l.value != zNewline && l.value != zEOF { + switch l.value { + case zString: + if l.token == "@" { + xs = append(xs, o) + l = <-c + continue + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad HIP RendezvousServers", l}, "" + } + if l.token[l.length-1] != '.' { + l.token = appendOrigin(l.token, o) + } + xs = append(xs, l.token) + case zBlank: + // Ok + default: + return nil, &ParseError{f, "bad HIP RendezvousServers", l}, "" + } + l = <-c + } + rr.RendezvousServers = xs + return rr, nil, l.comment +} + +func setCERT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(CERT) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + if v, ok := StringToCertType[l.token]; ok { + rr.Type = v + } else if i, e := strconv.ParseUint(l.token, 10, 16); e != nil { + return nil, &ParseError{f, "bad CERT Type", l}, "" + } else { + rr.Type = uint16(i) + } + <-c // zBlank + l = <-c // zString + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad CERT KeyTag", l}, "" + } + rr.KeyTag = uint16(i) + <-c // zBlank + l = <-c // zString + if v, ok := StringToAlgorithm[l.token]; ok { + rr.Algorithm = v + } else if i, e := strconv.ParseUint(l.token, 10, 8); e != nil { + return nil, &ParseError{f, "bad CERT Algorithm", l}, "" + } else { + rr.Algorithm = uint8(i) + } + s, e1, c1 := endingToString(c, "bad CERT Certificate", f) + if e1 != nil { + return nil, e1, c1 + } + rr.Certificate = s + return rr, nil, c1 +} + +func setOPENPGPKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(OPENPGPKEY) + rr.Hdr = h + + s, e, c1 := endingToString(c, "bad OPENPGPKEY PublicKey", f) + if e != nil { + return nil, e, c1 + } + rr.PublicKey = s + return rr, nil, c1 +} + +func setSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + r, e, s := setRRSIG(h, c, o, f) + if r != nil { + return &SIG{*r.(*RRSIG)}, e, s + } + return nil, e, s +} + +func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(RRSIG) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + if t, ok := StringToType[l.tokenUpper]; !ok { + if strings.HasPrefix(l.tokenUpper, "TYPE") { + t, ok = typeToInt(l.tokenUpper) + if !ok { + return nil, &ParseError{f, "bad RRSIG Typecovered", l}, "" + } + rr.TypeCovered = t + } else { + return nil, &ParseError{f, "bad RRSIG Typecovered", l}, "" + } + } else { + rr.TypeCovered = t + } + <-c // zBlank + l = <-c + i, err := strconv.ParseUint(l.token, 10, 8) + if err != nil || l.err { + return nil, &ParseError{f, "bad RRSIG Algorithm", l}, "" + } + rr.Algorithm = uint8(i) + <-c // zBlank + l = <-c + i, err = strconv.ParseUint(l.token, 10, 8) + if err != nil || l.err { + return nil, &ParseError{f, "bad RRSIG Labels", l}, "" + } + rr.Labels = uint8(i) + <-c // zBlank + l = <-c + i, err = strconv.ParseUint(l.token, 10, 32) + if err != nil || l.err { + return nil, &ParseError{f, "bad RRSIG OrigTtl", l}, "" + } + rr.OrigTtl = uint32(i) + <-c // zBlank + l = <-c + if i, err := StringToTime(l.token); err != nil { + // Try to see if all numeric and use it as epoch + if i, err := strconv.ParseInt(l.token, 10, 64); err == nil { + // TODO(miek): error out on > MAX_UINT32, same below + rr.Expiration = uint32(i) + } else { + return nil, &ParseError{f, "bad RRSIG Expiration", l}, "" + } + } else { + rr.Expiration = i + } + <-c // zBlank + l = <-c + if i, err := StringToTime(l.token); err != nil { + if i, err := strconv.ParseInt(l.token, 10, 64); err == nil { + rr.Inception = uint32(i) + } else { + return nil, &ParseError{f, "bad RRSIG Inception", l}, "" + } + } else { + rr.Inception = i + } + <-c // zBlank + l = <-c + i, err = strconv.ParseUint(l.token, 10, 16) + if err != nil || l.err { + return nil, &ParseError{f, "bad RRSIG KeyTag", l}, "" + } + rr.KeyTag = uint16(i) + <-c // zBlank + l = <-c + rr.SignerName = l.token + if l.token == "@" { + rr.SignerName = o + } else { + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad RRSIG SignerName", l}, "" + } + if rr.SignerName[l.length-1] != '.' { + rr.SignerName = appendOrigin(rr.SignerName, o) + } + } + s, e, c1 := endingToString(c, "bad RRSIG Signature", f) + if e != nil { + return nil, e, c1 + } + rr.Signature = s + return rr, nil, c1 +} + +func setNSEC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NSEC) + rr.Hdr = h + + l := <-c + rr.NextDomain = l.token + if l.length == 0 { + return rr, nil, l.comment + } + if l.token == "@" { + rr.NextDomain = o + } else { + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad NSEC NextDomain", l}, "" + } + if rr.NextDomain[l.length-1] != '.' { + rr.NextDomain = appendOrigin(rr.NextDomain, o) + } + } + + rr.TypeBitMap = make([]uint16, 0) + var ( + k uint16 + ok bool + ) + l = <-c + for l.value != zNewline && l.value != zEOF { + switch l.value { + case zBlank: + // Ok + case zString: + if k, ok = StringToType[l.tokenUpper]; !ok { + if k, ok = typeToInt(l.tokenUpper); !ok { + return nil, &ParseError{f, "bad NSEC TypeBitMap", l}, "" + } + } + rr.TypeBitMap = append(rr.TypeBitMap, k) + default: + return nil, &ParseError{f, "bad NSEC TypeBitMap", l}, "" + } + l = <-c + } + return rr, nil, l.comment +} + +func setNSEC3(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NSEC3) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad NSEC3 Hash", l}, "" + } + rr.Hash = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad NSEC3 Flags", l}, "" + } + rr.Flags = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad NSEC3 Iterations", l}, "" + } + rr.Iterations = uint16(i) + <-c + l = <-c + if len(l.token) == 0 || l.err { + return nil, &ParseError{f, "bad NSEC3 Salt", l}, "" + } + rr.SaltLength = uint8(len(l.token)) / 2 + rr.Salt = l.token + + <-c + l = <-c + if len(l.token) == 0 || l.err { + return nil, &ParseError{f, "bad NSEC3 NextDomain", l}, "" + } + rr.HashLength = 20 // Fix for NSEC3 (sha1 160 bits) + rr.NextDomain = l.token + + rr.TypeBitMap = make([]uint16, 0) + var ( + k uint16 + ok bool + ) + l = <-c + for l.value != zNewline && l.value != zEOF { + switch l.value { + case zBlank: + // Ok + case zString: + if k, ok = StringToType[l.tokenUpper]; !ok { + if k, ok = typeToInt(l.tokenUpper); !ok { + return nil, &ParseError{f, "bad NSEC3 TypeBitMap", l}, "" + } + } + rr.TypeBitMap = append(rr.TypeBitMap, k) + default: + return nil, &ParseError{f, "bad NSEC3 TypeBitMap", l}, "" + } + l = <-c + } + return rr, nil, l.comment +} + +func setNSEC3PARAM(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NSEC3PARAM) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad NSEC3PARAM Hash", l}, "" + } + rr.Hash = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad NSEC3PARAM Flags", l}, "" + } + rr.Flags = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad NSEC3PARAM Iterations", l}, "" + } + rr.Iterations = uint16(i) + <-c + l = <-c + rr.SaltLength = uint8(len(l.token)) + rr.Salt = l.token + return rr, nil, "" +} + +func setEUI48(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(EUI48) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + if l.length != 17 || l.err { + return nil, &ParseError{f, "bad EUI48 Address", l}, "" + } + addr := make([]byte, 12) + dash := 0 + for i := 0; i < 10; i += 2 { + addr[i] = l.token[i+dash] + addr[i+1] = l.token[i+1+dash] + dash++ + if l.token[i+1+dash] != '-' { + return nil, &ParseError{f, "bad EUI48 Address", l}, "" + } + } + addr[10] = l.token[15] + addr[11] = l.token[16] + + i, e := strconv.ParseUint(string(addr), 16, 48) + if e != nil { + return nil, &ParseError{f, "bad EUI48 Address", l}, "" + } + rr.Address = i + return rr, nil, "" +} + +func setEUI64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(EUI64) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + if l.length != 23 || l.err { + return nil, &ParseError{f, "bad EUI64 Address", l}, "" + } + addr := make([]byte, 16) + dash := 0 + for i := 0; i < 14; i += 2 { + addr[i] = l.token[i+dash] + addr[i+1] = l.token[i+1+dash] + dash++ + if l.token[i+1+dash] != '-' { + return nil, &ParseError{f, "bad EUI64 Address", l}, "" + } + } + addr[14] = l.token[21] + addr[15] = l.token[22] + + i, e := strconv.ParseUint(string(addr), 16, 64) + if e != nil { + return nil, &ParseError{f, "bad EUI68 Address", l}, "" + } + rr.Address = uint64(i) + return rr, nil, "" +} + +func setSSHFP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(SSHFP) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad SSHFP Algorithm", l}, "" + } + rr.Algorithm = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad SSHFP Type", l}, "" + } + rr.Type = uint8(i) + <-c // zBlank + s, e1, c1 := endingToString(c, "bad SSHFP Fingerprint", f) + if e1 != nil { + return nil, e1, c1 + } + rr.FingerPrint = s + return rr, nil, "" +} + +func setDNSKEYs(h RR_Header, c chan lex, o, f, typ string) (RR, *ParseError, string) { + rr := new(DNSKEY) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad " + typ + " Flags", l}, "" + } + rr.Flags = uint16(i) + <-c // zBlank + l = <-c // zString + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad " + typ + " Protocol", l}, "" + } + rr.Protocol = uint8(i) + <-c // zBlank + l = <-c // zString + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad " + typ + " Algorithm", l}, "" + } + rr.Algorithm = uint8(i) + s, e1, c1 := endingToString(c, "bad "+typ+" PublicKey", f) + if e1 != nil { + return nil, e1, c1 + } + rr.PublicKey = s + return rr, nil, c1 +} + +func setKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + r, e, s := setDNSKEYs(h, c, o, f, "KEY") + if r != nil { + return &KEY{*r.(*DNSKEY)}, e, s + } + return nil, e, s +} + +func setDNSKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + r, e, s := setDNSKEYs(h, c, o, f, "DNSKEY") + return r, e, s +} + +func setCDNSKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + r, e, s := setDNSKEYs(h, c, o, f, "CDNSKEY") + if r != nil { + return &CDNSKEY{*r.(*DNSKEY)}, e, s + } + return nil, e, s +} + +func setRKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(RKEY) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad RKEY Flags", l}, "" + } + rr.Flags = uint16(i) + <-c // zBlank + l = <-c // zString + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad RKEY Protocol", l}, "" + } + rr.Protocol = uint8(i) + <-c // zBlank + l = <-c // zString + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad RKEY Algorithm", l}, "" + } + rr.Algorithm = uint8(i) + s, e1, c1 := endingToString(c, "bad RKEY PublicKey", f) + if e1 != nil { + return nil, e1, c1 + } + rr.PublicKey = s + return rr, nil, c1 +} + +func setEID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(EID) + rr.Hdr = h + s, e, c1 := endingToString(c, "bad EID Endpoint", f) + if e != nil { + return nil, e, c1 + } + rr.Endpoint = s + return rr, nil, c1 +} + +func setNIMLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NIMLOC) + rr.Hdr = h + s, e, c1 := endingToString(c, "bad NIMLOC Locator", f) + if e != nil { + return nil, e, c1 + } + rr.Locator = s + return rr, nil, c1 +} + +func setGPOS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(GPOS) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, "" + } + _, e := strconv.ParseFloat(l.token, 64) + if e != nil || l.err { + return nil, &ParseError{f, "bad GPOS Longitude", l}, "" + } + rr.Longitude = l.token + <-c // zBlank + l = <-c + _, e = strconv.ParseFloat(l.token, 64) + if e != nil || l.err { + return nil, &ParseError{f, "bad GPOS Latitude", l}, "" + } + rr.Latitude = l.token + <-c // zBlank + l = <-c + _, e = strconv.ParseFloat(l.token, 64) + if e != nil || l.err { + return nil, &ParseError{f, "bad GPOS Altitude", l}, "" + } + rr.Altitude = l.token + return rr, nil, "" +} + +func setDSs(h RR_Header, c chan lex, o, f, typ string) (RR, *ParseError, string) { + rr := new(DS) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad " + typ + " KeyTag", l}, "" + } + rr.KeyTag = uint16(i) + <-c // zBlank + l = <-c + if i, e = strconv.ParseUint(l.token, 10, 8); e != nil { + i, ok := StringToAlgorithm[l.tokenUpper] + if !ok || l.err { + return nil, &ParseError{f, "bad " + typ + " Algorithm", l}, "" + } + rr.Algorithm = i + } else { + rr.Algorithm = uint8(i) + } + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad " + typ + " DigestType", l}, "" + } + rr.DigestType = uint8(i) + s, e1, c1 := endingToString(c, "bad "+typ+" Digest", f) + if e1 != nil { + return nil, e1, c1 + } + rr.Digest = s + return rr, nil, c1 +} + +func setDS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + r, e, s := setDSs(h, c, o, f, "DS") + return r, e, s +} + +func setDLV(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + r, e, s := setDSs(h, c, o, f, "DLV") + if r != nil { + return &DLV{*r.(*DS)}, e, s + } + return nil, e, s +} + +func setCDS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + r, e, s := setDSs(h, c, o, f, "CDS") + if r != nil { + return &CDS{*r.(*DS)}, e, s + } + return nil, e, s +} + +func setTA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(TA) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad TA KeyTag", l}, "" + } + rr.KeyTag = uint16(i) + <-c // zBlank + l = <-c + if i, e := strconv.ParseUint(l.token, 10, 8); e != nil { + i, ok := StringToAlgorithm[l.tokenUpper] + if !ok || l.err { + return nil, &ParseError{f, "bad TA Algorithm", l}, "" + } + rr.Algorithm = i + } else { + rr.Algorithm = uint8(i) + } + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad TA DigestType", l}, "" + } + rr.DigestType = uint8(i) + s, e, c1 := endingToString(c, "bad TA Digest", f) + if e != nil { + return nil, e.(*ParseError), c1 + } + rr.Digest = s + return rr, nil, c1 +} + +func setTLSA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(TLSA) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad TLSA Usage", l}, "" + } + rr.Usage = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad TLSA Selector", l}, "" + } + rr.Selector = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad TLSA MatchingType", l}, "" + } + rr.MatchingType = uint8(i) + // So this needs be e2 (i.e. different than e), because...??t + s, e2, c1 := endingToString(c, "bad TLSA Certificate", f) + if e2 != nil { + return nil, e2, c1 + } + rr.Certificate = s + return rr, nil, c1 +} + +func setSMIMEA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(SMIMEA) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, e := strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad SMIMEA Usage", l}, "" + } + rr.Usage = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad SMIMEA Selector", l}, "" + } + rr.Selector = uint8(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 8) + if e != nil || l.err { + return nil, &ParseError{f, "bad SMIMEA MatchingType", l}, "" + } + rr.MatchingType = uint8(i) + // So this needs be e2 (i.e. different than e), because...??t + s, e2, c1 := endingToString(c, "bad SMIMEA Certificate", f) + if e2 != nil { + return nil, e2, c1 + } + rr.Certificate = s + return rr, nil, c1 +} + +func setRFC3597(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(RFC3597) + rr.Hdr = h + l := <-c + if l.token != "\\#" { + return nil, &ParseError{f, "bad RFC3597 Rdata", l}, "" + } + <-c // zBlank + l = <-c + rdlength, e := strconv.Atoi(l.token) + if e != nil || l.err { + return nil, &ParseError{f, "bad RFC3597 Rdata ", l}, "" + } + + s, e1, c1 := endingToString(c, "bad RFC3597 Rdata", f) + if e1 != nil { + return nil, e1, c1 + } + if rdlength*2 != len(s) { + return nil, &ParseError{f, "bad RFC3597 Rdata", l}, "" + } + rr.Rdata = s + return rr, nil, c1 +} + +func setSPF(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(SPF) + rr.Hdr = h + + s, e, c1 := endingToTxtSlice(c, "bad SPF Txt", f) + if e != nil { + return nil, e, "" + } + rr.Txt = s + return rr, nil, c1 +} + +func setAVC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(AVC) + rr.Hdr = h + + s, e, c1 := endingToTxtSlice(c, "bad AVC Txt", f) + if e != nil { + return nil, e, "" + } + rr.Txt = s + return rr, nil, c1 +} + +func setTXT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(TXT) + rr.Hdr = h + + // no zBlank reading here, because all this rdata is TXT + s, e, c1 := endingToTxtSlice(c, "bad TXT Txt", f) + if e != nil { + return nil, e, "" + } + rr.Txt = s + return rr, nil, c1 +} + +// identical to setTXT +func setNINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NINFO) + rr.Hdr = h + + s, e, c1 := endingToTxtSlice(c, "bad NINFO ZSData", f) + if e != nil { + return nil, e, "" + } + rr.ZSData = s + return rr, nil, c1 +} + +func setURI(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(URI) + rr.Hdr = h + + l := <-c + if l.length == 0 { // Dynamic updates. + return rr, nil, "" + } + + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad URI Priority", l}, "" + } + rr.Priority = uint16(i) + <-c // zBlank + l = <-c + i, e = strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad URI Weight", l}, "" + } + rr.Weight = uint16(i) + + <-c // zBlank + s, err, c1 := endingToTxtSlice(c, "bad URI Target", f) + if err != nil { + return nil, err, "" + } + if len(s) > 1 { + return nil, &ParseError{f, "bad URI Target", l}, "" + } + rr.Target = s[0] + return rr, nil, c1 +} + +func setDHCID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + // awesome record to parse! + rr := new(DHCID) + rr.Hdr = h + + s, e, c1 := endingToString(c, "bad DHCID Digest", f) + if e != nil { + return nil, e, c1 + } + rr.Digest = s + return rr, nil, c1 +} + +func setNID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(NID) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad NID Preference", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + u, err := stringToNodeID(l) + if err != nil || l.err { + return nil, err, "" + } + rr.NodeID = u + return rr, nil, "" +} + +func setL32(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(L32) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad L32 Preference", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Locator32 = net.ParseIP(l.token) + if rr.Locator32 == nil || l.err { + return nil, &ParseError{f, "bad L32 Locator", l}, "" + } + return rr, nil, "" +} + +func setLP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(LP) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad LP Preference", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Fqdn = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Fqdn = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad LP Fqdn", l}, "" + } + if rr.Fqdn[l.length-1] != '.' { + rr.Fqdn = appendOrigin(rr.Fqdn, o) + } + return rr, nil, "" +} + +func setL64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(L64) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad L64 Preference", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + u, err := stringToNodeID(l) + if err != nil || l.err { + return nil, err, "" + } + rr.Locator64 = u + return rr, nil, "" +} + +func setUID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(UID) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 32) + if e != nil || l.err { + return nil, &ParseError{f, "bad UID Uid", l}, "" + } + rr.Uid = uint32(i) + return rr, nil, "" +} + +func setGID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(GID) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 32) + if e != nil || l.err { + return nil, &ParseError{f, "bad GID Gid", l}, "" + } + rr.Gid = uint32(i) + return rr, nil, "" +} + +func setUINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(UINFO) + rr.Hdr = h + s, e, c1 := endingToTxtSlice(c, "bad UINFO Uinfo", f) + if e != nil { + return nil, e, c1 + } + if ln := len(s); ln == 0 { + return rr, nil, c1 + } + rr.Uinfo = s[0] // silently discard anything after the first character-string + return rr, nil, c1 +} + +func setPX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(PX) + rr.Hdr = h + + l := <-c + if l.length == 0 { + return rr, nil, "" + } + i, e := strconv.ParseUint(l.token, 10, 16) + if e != nil || l.err { + return nil, &ParseError{f, "bad PX Preference", l}, "" + } + rr.Preference = uint16(i) + <-c // zBlank + l = <-c // zString + rr.Map822 = l.token + if l.length == 0 { + return rr, nil, "" + } + if l.token == "@" { + rr.Map822 = o + return rr, nil, "" + } + _, ok := IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad PX Map822", l}, "" + } + if rr.Map822[l.length-1] != '.' { + rr.Map822 = appendOrigin(rr.Map822, o) + } + <-c // zBlank + l = <-c // zString + rr.Mapx400 = l.token + if l.token == "@" { + rr.Mapx400 = o + return rr, nil, "" + } + _, ok = IsDomainName(l.token) + if !ok || l.length == 0 || l.err { + return nil, &ParseError{f, "bad PX Mapx400", l}, "" + } + if rr.Mapx400[l.length-1] != '.' { + rr.Mapx400 = appendOrigin(rr.Mapx400, o) + } + return rr, nil, "" +} + +func setCAA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(CAA) + rr.Hdr = h + l := <-c + if l.length == 0 { + return rr, nil, l.comment + } + i, err := strconv.ParseUint(l.token, 10, 8) + if err != nil || l.err { + return nil, &ParseError{f, "bad CAA Flag", l}, "" + } + rr.Flag = uint8(i) + + <-c // zBlank + l = <-c // zString + if l.value != zString { + return nil, &ParseError{f, "bad CAA Tag", l}, "" + } + rr.Tag = l.token + + <-c // zBlank + s, e, c1 := endingToTxtSlice(c, "bad CAA Value", f) + if e != nil { + return nil, e, "" + } + if len(s) > 1 { + return nil, &ParseError{f, "bad CAA Value", l}, "" + } + rr.Value = s[0] + return rr, nil, c1 +} + +var typeToparserFunc = map[uint16]parserFunc{ + TypeAAAA: {setAAAA, false}, + TypeAFSDB: {setAFSDB, false}, + TypeA: {setA, false}, + TypeCAA: {setCAA, true}, + TypeCDS: {setCDS, true}, + TypeCDNSKEY: {setCDNSKEY, true}, + TypeCERT: {setCERT, true}, + TypeCNAME: {setCNAME, false}, + TypeDHCID: {setDHCID, true}, + TypeDLV: {setDLV, true}, + TypeDNAME: {setDNAME, false}, + TypeKEY: {setKEY, true}, + TypeDNSKEY: {setDNSKEY, true}, + TypeDS: {setDS, true}, + TypeEID: {setEID, true}, + TypeEUI48: {setEUI48, false}, + TypeEUI64: {setEUI64, false}, + TypeGID: {setGID, false}, + TypeGPOS: {setGPOS, false}, + TypeHINFO: {setHINFO, true}, + TypeHIP: {setHIP, true}, + TypeKX: {setKX, false}, + TypeL32: {setL32, false}, + TypeL64: {setL64, false}, + TypeLOC: {setLOC, true}, + TypeLP: {setLP, false}, + TypeMB: {setMB, false}, + TypeMD: {setMD, false}, + TypeMF: {setMF, false}, + TypeMG: {setMG, false}, + TypeMINFO: {setMINFO, false}, + TypeMR: {setMR, false}, + TypeMX: {setMX, false}, + TypeNAPTR: {setNAPTR, false}, + TypeNID: {setNID, false}, + TypeNIMLOC: {setNIMLOC, true}, + TypeNINFO: {setNINFO, true}, + TypeNSAPPTR: {setNSAPPTR, false}, + TypeNSEC3PARAM: {setNSEC3PARAM, false}, + TypeNSEC3: {setNSEC3, true}, + TypeNSEC: {setNSEC, true}, + TypeNS: {setNS, false}, + TypeOPENPGPKEY: {setOPENPGPKEY, true}, + TypePTR: {setPTR, false}, + TypePX: {setPX, false}, + TypeSIG: {setSIG, true}, + TypeRKEY: {setRKEY, true}, + TypeRP: {setRP, false}, + TypeRRSIG: {setRRSIG, true}, + TypeRT: {setRT, false}, + TypeSMIMEA: {setSMIMEA, true}, + TypeSOA: {setSOA, false}, + TypeSPF: {setSPF, true}, + TypeAVC: {setAVC, true}, + TypeSRV: {setSRV, false}, + TypeSSHFP: {setSSHFP, true}, + TypeTALINK: {setTALINK, false}, + TypeTA: {setTA, true}, + TypeTLSA: {setTLSA, true}, + TypeTXT: {setTXT, true}, + TypeUID: {setUID, false}, + TypeUINFO: {setUINFO, true}, + TypeURI: {setURI, true}, + TypeX25: {setX25, false}, +} diff --git a/vendor/github.com/miekg/dns/scanner.go b/vendor/github.com/miekg/dns/scanner.go new file mode 100644 index 00000000..c29bc2f3 --- /dev/null +++ b/vendor/github.com/miekg/dns/scanner.go @@ -0,0 +1,43 @@ +package dns + +// Implement a simple scanner, return a byte stream from an io reader. + +import ( + "bufio" + "io" + "text/scanner" +) + +type scan struct { + src *bufio.Reader + position scanner.Position + eof bool // Have we just seen a eof +} + +func scanInit(r io.Reader) *scan { + s := new(scan) + s.src = bufio.NewReader(r) + s.position.Line = 1 + return s +} + +// tokenText returns the next byte from the input +func (s *scan) tokenText() (byte, error) { + c, err := s.src.ReadByte() + if err != nil { + return c, err + } + // delay the newline handling until the next token is delivered, + // fixes off-by-one errors when reporting a parse error. + if s.eof == true { + s.position.Line++ + s.position.Column = 0 + s.eof = false + } + if c == '\n' { + s.eof = true + return c, nil + } + s.position.Column++ + return c, nil +} diff --git a/vendor/github.com/miekg/dns/server.go b/vendor/github.com/miekg/dns/server.go new file mode 100644 index 00000000..0ca6e008 --- /dev/null +++ b/vendor/github.com/miekg/dns/server.go @@ -0,0 +1,734 @@ +// DNS server implementation. + +package dns + +import ( + "bytes" + "crypto/tls" + "encoding/binary" + "io" + "net" + "sync" + "time" +) + +// Maximum number of TCP queries before we close the socket. +const maxTCPQueries = 128 + +// Handler is implemented by any value that implements ServeDNS. +type Handler interface { + ServeDNS(w ResponseWriter, r *Msg) +} + +// A ResponseWriter interface is used by an DNS handler to +// construct an DNS response. +type ResponseWriter interface { + // LocalAddr returns the net.Addr of the server + LocalAddr() net.Addr + // RemoteAddr returns the net.Addr of the client that sent the current request. + RemoteAddr() net.Addr + // WriteMsg writes a reply back to the client. + WriteMsg(*Msg) error + // Write writes a raw buffer back to the client. + Write([]byte) (int, error) + // Close closes the connection. + Close() error + // TsigStatus returns the status of the Tsig. + TsigStatus() error + // TsigTimersOnly sets the tsig timers only boolean. + TsigTimersOnly(bool) + // Hijack lets the caller take over the connection. + // After a call to Hijack(), the DNS package will not do anything with the connection. + Hijack() +} + +type response struct { + hijacked bool // connection has been hijacked by handler + tsigStatus error + tsigTimersOnly bool + tsigRequestMAC string + tsigSecret map[string]string // the tsig secrets + udp *net.UDPConn // i/o connection if UDP was used + tcp net.Conn // i/o connection if TCP was used + udpSession *SessionUDP // oob data to get egress interface right + remoteAddr net.Addr // address of the client + writer Writer // writer to output the raw DNS bits +} + +// ServeMux is an DNS request multiplexer. It matches the +// zone name of each incoming request against a list of +// registered patterns add calls the handler for the pattern +// that most closely matches the zone name. ServeMux is DNSSEC aware, meaning +// that queries for the DS record are redirected to the parent zone (if that +// is also registered), otherwise the child gets the query. +// ServeMux is also safe for concurrent access from multiple goroutines. +type ServeMux struct { + z map[string]Handler + m *sync.RWMutex +} + +// NewServeMux allocates and returns a new ServeMux. +func NewServeMux() *ServeMux { return &ServeMux{z: make(map[string]Handler), m: new(sync.RWMutex)} } + +// DefaultServeMux is the default ServeMux used by Serve. +var DefaultServeMux = NewServeMux() + +// The HandlerFunc type is an adapter to allow the use of +// ordinary functions as DNS handlers. If f is a function +// with the appropriate signature, HandlerFunc(f) is a +// Handler object that calls f. +type HandlerFunc func(ResponseWriter, *Msg) + +// ServeDNS calls f(w, r). +func (f HandlerFunc) ServeDNS(w ResponseWriter, r *Msg) { + f(w, r) +} + +// HandleFailed returns a HandlerFunc that returns SERVFAIL for every request it gets. +func HandleFailed(w ResponseWriter, r *Msg) { + m := new(Msg) + m.SetRcode(r, RcodeServerFailure) + // does not matter if this write fails + w.WriteMsg(m) +} + +func failedHandler() Handler { return HandlerFunc(HandleFailed) } + +// ListenAndServe Starts a server on address and network specified Invoke handler +// for incoming queries. +func ListenAndServe(addr string, network string, handler Handler) error { + server := &Server{Addr: addr, Net: network, Handler: handler} + return server.ListenAndServe() +} + +// ListenAndServeTLS acts like http.ListenAndServeTLS, more information in +// http://golang.org/pkg/net/http/#ListenAndServeTLS +func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return err + } + + config := tls.Config{ + Certificates: []tls.Certificate{cert}, + } + + server := &Server{ + Addr: addr, + Net: "tcp-tls", + TLSConfig: &config, + Handler: handler, + } + + return server.ListenAndServe() +} + +// ActivateAndServe activates a server with a listener from systemd, +// l and p should not both be non-nil. +// If both l and p are not nil only p will be used. +// Invoke handler for incoming queries. +func ActivateAndServe(l net.Listener, p net.PacketConn, handler Handler) error { + server := &Server{Listener: l, PacketConn: p, Handler: handler} + return server.ActivateAndServe() +} + +func (mux *ServeMux) match(q string, t uint16) Handler { + mux.m.RLock() + defer mux.m.RUnlock() + var handler Handler + b := make([]byte, len(q)) // worst case, one label of length q + off := 0 + end := false + for { + l := len(q[off:]) + for i := 0; i < l; i++ { + b[i] = q[off+i] + if b[i] >= 'A' && b[i] <= 'Z' { + b[i] |= ('a' - 'A') + } + } + if h, ok := mux.z[string(b[:l])]; ok { // causes garbage, might want to change the map key + if t != TypeDS { + return h + } + // Continue for DS to see if we have a parent too, if so delegeate to the parent + handler = h + } + off, end = NextLabel(q, off) + if end { + break + } + } + // Wildcard match, if we have found nothing try the root zone as a last resort. + if h, ok := mux.z["."]; ok { + return h + } + return handler +} + +// Handle adds a handler to the ServeMux for pattern. +func (mux *ServeMux) Handle(pattern string, handler Handler) { + if pattern == "" { + panic("dns: invalid pattern " + pattern) + } + mux.m.Lock() + mux.z[Fqdn(pattern)] = handler + mux.m.Unlock() +} + +// HandleFunc adds a handler function to the ServeMux for pattern. +func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Msg)) { + mux.Handle(pattern, HandlerFunc(handler)) +} + +// HandleRemove deregistrars the handler specific for pattern from the ServeMux. +func (mux *ServeMux) HandleRemove(pattern string) { + if pattern == "" { + panic("dns: invalid pattern " + pattern) + } + mux.m.Lock() + delete(mux.z, Fqdn(pattern)) + mux.m.Unlock() +} + +// ServeDNS dispatches the request to the handler whose +// pattern most closely matches the request message. If DefaultServeMux +// is used the correct thing for DS queries is done: a possible parent +// is sought. +// If no handler is found a standard SERVFAIL message is returned +// If the request message does not have exactly one question in the +// question section a SERVFAIL is returned, unlesss Unsafe is true. +func (mux *ServeMux) ServeDNS(w ResponseWriter, request *Msg) { + var h Handler + if len(request.Question) < 1 { // allow more than one question + h = failedHandler() + } else { + if h = mux.match(request.Question[0].Name, request.Question[0].Qtype); h == nil { + h = failedHandler() + } + } + h.ServeDNS(w, request) +} + +// Handle registers the handler with the given pattern +// in the DefaultServeMux. The documentation for +// ServeMux explains how patterns are matched. +func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } + +// HandleRemove deregisters the handle with the given pattern +// in the DefaultServeMux. +func HandleRemove(pattern string) { DefaultServeMux.HandleRemove(pattern) } + +// HandleFunc registers the handler function with the given pattern +// in the DefaultServeMux. +func HandleFunc(pattern string, handler func(ResponseWriter, *Msg)) { + DefaultServeMux.HandleFunc(pattern, handler) +} + +// Writer writes raw DNS messages; each call to Write should send an entire message. +type Writer interface { + io.Writer +} + +// Reader reads raw DNS messages; each call to ReadTCP or ReadUDP should return an entire message. +type Reader interface { + // ReadTCP reads a raw message from a TCP connection. Implementations may alter + // connection properties, for example the read-deadline. + ReadTCP(conn net.Conn, timeout time.Duration) ([]byte, error) + // ReadUDP reads a raw message from a UDP connection. Implementations may alter + // connection properties, for example the read-deadline. + ReadUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *SessionUDP, error) +} + +// defaultReader is an adapter for the Server struct that implements the Reader interface +// using the readTCP and readUDP func of the embedded Server. +type defaultReader struct { + *Server +} + +func (dr *defaultReader) ReadTCP(conn net.Conn, timeout time.Duration) ([]byte, error) { + return dr.readTCP(conn, timeout) +} + +func (dr *defaultReader) ReadUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *SessionUDP, error) { + return dr.readUDP(conn, timeout) +} + +// DecorateReader is a decorator hook for extending or supplanting the functionality of a Reader. +// Implementations should never return a nil Reader. +type DecorateReader func(Reader) Reader + +// DecorateWriter is a decorator hook for extending or supplanting the functionality of a Writer. +// Implementations should never return a nil Writer. +type DecorateWriter func(Writer) Writer + +// A Server defines parameters for running an DNS server. +type Server struct { + // Address to listen on, ":dns" if empty. + Addr string + // if "tcp" or "tcp-tls" (DNS over TLS) it will invoke a TCP listener, otherwise an UDP one + Net string + // TCP Listener to use, this is to aid in systemd's socket activation. + Listener net.Listener + // TLS connection configuration + TLSConfig *tls.Config + // UDP "Listener" to use, this is to aid in systemd's socket activation. + PacketConn net.PacketConn + // Handler to invoke, dns.DefaultServeMux if nil. + Handler Handler + // Default buffer size to use to read incoming UDP messages. If not set + // it defaults to MinMsgSize (512 B). + UDPSize int + // The net.Conn.SetReadTimeout value for new connections, defaults to 2 * time.Second. + ReadTimeout time.Duration + // The net.Conn.SetWriteTimeout value for new connections, defaults to 2 * time.Second. + WriteTimeout time.Duration + // TCP idle timeout for multiple queries, if nil, defaults to 8 * time.Second (RFC 5966). + IdleTimeout func() time.Duration + // Secret(s) for Tsig map[]. + TsigSecret map[string]string + // Unsafe instructs the server to disregard any sanity checks and directly hand the message to + // the handler. It will specifically not check if the query has the QR bit not set. + Unsafe bool + // If NotifyStartedFunc is set it is called once the server has started listening. + NotifyStartedFunc func() + // DecorateReader is optional, allows customization of the process that reads raw DNS messages. + DecorateReader DecorateReader + // DecorateWriter is optional, allows customization of the process that writes raw DNS messages. + DecorateWriter DecorateWriter + + // Graceful shutdown handling + + inFlight sync.WaitGroup + + lock sync.RWMutex + started bool +} + +// ListenAndServe starts a nameserver on the configured address in *Server. +func (srv *Server) ListenAndServe() error { + srv.lock.Lock() + defer srv.lock.Unlock() + if srv.started { + return &Error{err: "server already started"} + } + addr := srv.Addr + if addr == "" { + addr = ":domain" + } + if srv.UDPSize == 0 { + srv.UDPSize = MinMsgSize + } + switch srv.Net { + case "tcp", "tcp4", "tcp6": + a, err := net.ResolveTCPAddr(srv.Net, addr) + if err != nil { + return err + } + l, err := net.ListenTCP(srv.Net, a) + if err != nil { + return err + } + srv.Listener = l + srv.started = true + srv.lock.Unlock() + err = srv.serveTCP(l) + srv.lock.Lock() // to satisfy the defer at the top + return err + case "tcp-tls", "tcp4-tls", "tcp6-tls": + network := "tcp" + if srv.Net == "tcp4-tls" { + network = "tcp4" + } else if srv.Net == "tcp6-tls" { + network = "tcp6" + } + + l, err := tls.Listen(network, addr, srv.TLSConfig) + if err != nil { + return err + } + srv.Listener = l + srv.started = true + srv.lock.Unlock() + err = srv.serveTCP(l) + srv.lock.Lock() // to satisfy the defer at the top + return err + case "udp", "udp4", "udp6": + a, err := net.ResolveUDPAddr(srv.Net, addr) + if err != nil { + return err + } + l, err := net.ListenUDP(srv.Net, a) + if err != nil { + return err + } + if e := setUDPSocketOptions(l); e != nil { + return e + } + srv.PacketConn = l + srv.started = true + srv.lock.Unlock() + err = srv.serveUDP(l) + srv.lock.Lock() // to satisfy the defer at the top + return err + } + return &Error{err: "bad network"} +} + +// ActivateAndServe starts a nameserver with the PacketConn or Listener +// configured in *Server. Its main use is to start a server from systemd. +func (srv *Server) ActivateAndServe() error { + srv.lock.Lock() + defer srv.lock.Unlock() + if srv.started { + return &Error{err: "server already started"} + } + pConn := srv.PacketConn + l := srv.Listener + if pConn != nil { + if srv.UDPSize == 0 { + srv.UDPSize = MinMsgSize + } + // Check PacketConn interface's type is valid and value + // is not nil + if t, ok := pConn.(*net.UDPConn); ok && t != nil { + if e := setUDPSocketOptions(t); e != nil { + return e + } + srv.started = true + srv.lock.Unlock() + e := srv.serveUDP(t) + srv.lock.Lock() // to satisfy the defer at the top + return e + } + } + if l != nil { + srv.started = true + srv.lock.Unlock() + e := srv.serveTCP(l) + srv.lock.Lock() // to satisfy the defer at the top + return e + } + return &Error{err: "bad listeners"} +} + +// Shutdown gracefully shuts down a server. After a call to Shutdown, ListenAndServe and +// ActivateAndServe will return. All in progress queries are completed before the server +// is taken down. If the Shutdown is taking longer than the reading timeout an error +// is returned. +func (srv *Server) Shutdown() error { + srv.lock.Lock() + if !srv.started { + srv.lock.Unlock() + return &Error{err: "server not started"} + } + srv.started = false + srv.lock.Unlock() + + if srv.PacketConn != nil { + srv.PacketConn.Close() + } + if srv.Listener != nil { + srv.Listener.Close() + } + + fin := make(chan bool) + go func() { + srv.inFlight.Wait() + fin <- true + }() + + select { + case <-time.After(srv.getReadTimeout()): + return &Error{err: "server shutdown is pending"} + case <-fin: + return nil + } +} + +// getReadTimeout is a helper func to use system timeout if server did not intend to change it. +func (srv *Server) getReadTimeout() time.Duration { + rtimeout := dnsTimeout + if srv.ReadTimeout != 0 { + rtimeout = srv.ReadTimeout + } + return rtimeout +} + +// serveTCP starts a TCP listener for the server. +// Each request is handled in a separate goroutine. +func (srv *Server) serveTCP(l net.Listener) error { + defer l.Close() + + if srv.NotifyStartedFunc != nil { + srv.NotifyStartedFunc() + } + + reader := Reader(&defaultReader{srv}) + if srv.DecorateReader != nil { + reader = srv.DecorateReader(reader) + } + + handler := srv.Handler + if handler == nil { + handler = DefaultServeMux + } + rtimeout := srv.getReadTimeout() + // deadline is not used here + for { + rw, err := l.Accept() + if err != nil { + if neterr, ok := err.(net.Error); ok && neterr.Temporary() { + continue + } + return err + } + m, err := reader.ReadTCP(rw, rtimeout) + srv.lock.RLock() + if !srv.started { + srv.lock.RUnlock() + return nil + } + srv.lock.RUnlock() + if err != nil { + continue + } + srv.inFlight.Add(1) + go srv.serve(rw.RemoteAddr(), handler, m, nil, nil, rw) + } +} + +// serveUDP starts a UDP listener for the server. +// Each request is handled in a separate goroutine. +func (srv *Server) serveUDP(l *net.UDPConn) error { + defer l.Close() + + if srv.NotifyStartedFunc != nil { + srv.NotifyStartedFunc() + } + + reader := Reader(&defaultReader{srv}) + if srv.DecorateReader != nil { + reader = srv.DecorateReader(reader) + } + + handler := srv.Handler + if handler == nil { + handler = DefaultServeMux + } + rtimeout := srv.getReadTimeout() + // deadline is not used here + for { + m, s, err := reader.ReadUDP(l, rtimeout) + srv.lock.RLock() + if !srv.started { + srv.lock.RUnlock() + return nil + } + srv.lock.RUnlock() + if err != nil { + continue + } + srv.inFlight.Add(1) + go srv.serve(s.RemoteAddr(), handler, m, l, s, nil) + } +} + +// Serve a new connection. +func (srv *Server) serve(a net.Addr, h Handler, m []byte, u *net.UDPConn, s *SessionUDP, t net.Conn) { + defer srv.inFlight.Done() + + w := &response{tsigSecret: srv.TsigSecret, udp: u, tcp: t, remoteAddr: a, udpSession: s} + if srv.DecorateWriter != nil { + w.writer = srv.DecorateWriter(w) + } else { + w.writer = w + } + + q := 0 // counter for the amount of TCP queries we get + + reader := Reader(&defaultReader{srv}) + if srv.DecorateReader != nil { + reader = srv.DecorateReader(reader) + } +Redo: + req := new(Msg) + err := req.Unpack(m) + if err != nil { // Send a FormatError back + x := new(Msg) + x.SetRcodeFormatError(req) + w.WriteMsg(x) + goto Exit + } + if !srv.Unsafe && req.Response { + goto Exit + } + + w.tsigStatus = nil + if w.tsigSecret != nil { + if t := req.IsTsig(); t != nil { + secret := t.Hdr.Name + if _, ok := w.tsigSecret[secret]; !ok { + w.tsigStatus = ErrKeyAlg + } + w.tsigStatus = TsigVerify(m, w.tsigSecret[secret], "", false) + w.tsigTimersOnly = false + w.tsigRequestMAC = req.Extra[len(req.Extra)-1].(*TSIG).MAC + } + } + h.ServeDNS(w, req) // Writes back to the client + +Exit: + if w.tcp == nil { + return + } + // TODO(miek): make this number configurable? + if q > maxTCPQueries { // close socket after this many queries + w.Close() + return + } + + if w.hijacked { + return // client calls Close() + } + if u != nil { // UDP, "close" and return + w.Close() + return + } + idleTimeout := tcpIdleTimeout + if srv.IdleTimeout != nil { + idleTimeout = srv.IdleTimeout() + } + m, err = reader.ReadTCP(w.tcp, idleTimeout) + if err == nil { + q++ + goto Redo + } + w.Close() + return +} + +func (srv *Server) readTCP(conn net.Conn, timeout time.Duration) ([]byte, error) { + conn.SetReadDeadline(time.Now().Add(timeout)) + l := make([]byte, 2) + n, err := conn.Read(l) + if err != nil || n != 2 { + if err != nil { + return nil, err + } + return nil, ErrShortRead + } + length := binary.BigEndian.Uint16(l) + if length == 0 { + return nil, ErrShortRead + } + m := make([]byte, int(length)) + n, err = conn.Read(m[:int(length)]) + if err != nil || n == 0 { + if err != nil { + return nil, err + } + return nil, ErrShortRead + } + i := n + for i < int(length) { + j, err := conn.Read(m[i:int(length)]) + if err != nil { + return nil, err + } + i += j + } + n = i + m = m[:n] + return m, nil +} + +func (srv *Server) readUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *SessionUDP, error) { + conn.SetReadDeadline(time.Now().Add(timeout)) + m := make([]byte, srv.UDPSize) + n, s, err := ReadFromSessionUDP(conn, m) + if err != nil || n == 0 { + if err != nil { + return nil, nil, err + } + return nil, nil, ErrShortRead + } + m = m[:n] + return m, s, nil +} + +// WriteMsg implements the ResponseWriter.WriteMsg method. +func (w *response) WriteMsg(m *Msg) (err error) { + var data []byte + if w.tsigSecret != nil { // if no secrets, dont check for the tsig (which is a longer check) + if t := m.IsTsig(); t != nil { + data, w.tsigRequestMAC, err = TsigGenerate(m, w.tsigSecret[t.Hdr.Name], w.tsigRequestMAC, w.tsigTimersOnly) + if err != nil { + return err + } + _, err = w.writer.Write(data) + return err + } + } + data, err = m.Pack() + if err != nil { + return err + } + _, err = w.writer.Write(data) + return err +} + +// Write implements the ResponseWriter.Write method. +func (w *response) Write(m []byte) (int, error) { + switch { + case w.udp != nil: + n, err := WriteToSessionUDP(w.udp, m, w.udpSession) + return n, err + case w.tcp != nil: + lm := len(m) + if lm < 2 { + return 0, io.ErrShortBuffer + } + if lm > MaxMsgSize { + return 0, &Error{err: "message too large"} + } + l := make([]byte, 2, 2+lm) + binary.BigEndian.PutUint16(l, uint16(lm)) + m = append(l, m...) + + n, err := io.Copy(w.tcp, bytes.NewReader(m)) + return int(n), err + } + panic("not reached") +} + +// LocalAddr implements the ResponseWriter.LocalAddr method. +func (w *response) LocalAddr() net.Addr { + if w.tcp != nil { + return w.tcp.LocalAddr() + } + return w.udp.LocalAddr() +} + +// RemoteAddr implements the ResponseWriter.RemoteAddr method. +func (w *response) RemoteAddr() net.Addr { return w.remoteAddr } + +// TsigStatus implements the ResponseWriter.TsigStatus method. +func (w *response) TsigStatus() error { return w.tsigStatus } + +// TsigTimersOnly implements the ResponseWriter.TsigTimersOnly method. +func (w *response) TsigTimersOnly(b bool) { w.tsigTimersOnly = b } + +// Hijack implements the ResponseWriter.Hijack method. +func (w *response) Hijack() { w.hijacked = true } + +// Close implements the ResponseWriter.Close method +func (w *response) Close() error { + // Can't close the udp conn, as that is actually the listener. + if w.tcp != nil { + e := w.tcp.Close() + w.tcp = nil + return e + } + return nil +} diff --git a/vendor/github.com/miekg/dns/sig0.go b/vendor/github.com/miekg/dns/sig0.go new file mode 100644 index 00000000..f31e9e68 --- /dev/null +++ b/vendor/github.com/miekg/dns/sig0.go @@ -0,0 +1,218 @@ +package dns + +import ( + "crypto" + "crypto/dsa" + "crypto/ecdsa" + "crypto/rsa" + "encoding/binary" + "math/big" + "strings" + "time" +) + +// Sign signs a dns.Msg. It fills the signature with the appropriate data. +// The SIG record should have the SignerName, KeyTag, Algorithm, Inception +// and Expiration set. +func (rr *SIG) Sign(k crypto.Signer, m *Msg) ([]byte, error) { + if k == nil { + return nil, ErrPrivKey + } + if rr.KeyTag == 0 || len(rr.SignerName) == 0 || rr.Algorithm == 0 { + return nil, ErrKey + } + rr.Header().Rrtype = TypeSIG + rr.Header().Class = ClassANY + rr.Header().Ttl = 0 + rr.Header().Name = "." + rr.OrigTtl = 0 + rr.TypeCovered = 0 + rr.Labels = 0 + + buf := make([]byte, m.Len()+rr.len()) + mbuf, err := m.PackBuffer(buf) + if err != nil { + return nil, err + } + if &buf[0] != &mbuf[0] { + return nil, ErrBuf + } + off, err := PackRR(rr, buf, len(mbuf), nil, false) + if err != nil { + return nil, err + } + buf = buf[:off:cap(buf)] + + hash, ok := AlgorithmToHash[rr.Algorithm] + if !ok { + return nil, ErrAlg + } + + hasher := hash.New() + // Write SIG rdata + hasher.Write(buf[len(mbuf)+1+2+2+4+2:]) + // Write message + hasher.Write(buf[:len(mbuf)]) + + signature, err := sign(k, hasher.Sum(nil), hash, rr.Algorithm) + if err != nil { + return nil, err + } + + rr.Signature = toBase64(signature) + + buf = append(buf, signature...) + if len(buf) > int(^uint16(0)) { + return nil, ErrBuf + } + // Adjust sig data length + rdoff := len(mbuf) + 1 + 2 + 2 + 4 + rdlen := binary.BigEndian.Uint16(buf[rdoff:]) + rdlen += uint16(len(signature)) + binary.BigEndian.PutUint16(buf[rdoff:], rdlen) + // Adjust additional count + adc := binary.BigEndian.Uint16(buf[10:]) + adc++ + binary.BigEndian.PutUint16(buf[10:], adc) + return buf, nil +} + +// Verify validates the message buf using the key k. +// It's assumed that buf is a valid message from which rr was unpacked. +func (rr *SIG) Verify(k *KEY, buf []byte) error { + if k == nil { + return ErrKey + } + if rr.KeyTag == 0 || len(rr.SignerName) == 0 || rr.Algorithm == 0 { + return ErrKey + } + + var hash crypto.Hash + switch rr.Algorithm { + case DSA, RSASHA1: + hash = crypto.SHA1 + case RSASHA256, ECDSAP256SHA256: + hash = crypto.SHA256 + case ECDSAP384SHA384: + hash = crypto.SHA384 + case RSASHA512: + hash = crypto.SHA512 + default: + return ErrAlg + } + hasher := hash.New() + + buflen := len(buf) + qdc := binary.BigEndian.Uint16(buf[4:]) + anc := binary.BigEndian.Uint16(buf[6:]) + auc := binary.BigEndian.Uint16(buf[8:]) + adc := binary.BigEndian.Uint16(buf[10:]) + offset := 12 + var err error + for i := uint16(0); i < qdc && offset < buflen; i++ { + _, offset, err = UnpackDomainName(buf, offset) + if err != nil { + return err + } + // Skip past Type and Class + offset += 2 + 2 + } + for i := uint16(1); i < anc+auc+adc && offset < buflen; i++ { + _, offset, err = UnpackDomainName(buf, offset) + if err != nil { + return err + } + // Skip past Type, Class and TTL + offset += 2 + 2 + 4 + if offset+1 >= buflen { + continue + } + var rdlen uint16 + rdlen = binary.BigEndian.Uint16(buf[offset:]) + offset += 2 + offset += int(rdlen) + } + if offset >= buflen { + return &Error{err: "overflowing unpacking signed message"} + } + + // offset should be just prior to SIG + bodyend := offset + // owner name SHOULD be root + _, offset, err = UnpackDomainName(buf, offset) + if err != nil { + return err + } + // Skip Type, Class, TTL, RDLen + offset += 2 + 2 + 4 + 2 + sigstart := offset + // Skip Type Covered, Algorithm, Labels, Original TTL + offset += 2 + 1 + 1 + 4 + if offset+4+4 >= buflen { + return &Error{err: "overflow unpacking signed message"} + } + expire := binary.BigEndian.Uint32(buf[offset:]) + offset += 4 + incept := binary.BigEndian.Uint32(buf[offset:]) + offset += 4 + now := uint32(time.Now().Unix()) + if now < incept || now > expire { + return ErrTime + } + // Skip key tag + offset += 2 + var signername string + signername, offset, err = UnpackDomainName(buf, offset) + if err != nil { + return err + } + // If key has come from the DNS name compression might + // have mangled the case of the name + if strings.ToLower(signername) != strings.ToLower(k.Header().Name) { + return &Error{err: "signer name doesn't match key name"} + } + sigend := offset + hasher.Write(buf[sigstart:sigend]) + hasher.Write(buf[:10]) + hasher.Write([]byte{ + byte((adc - 1) << 8), + byte(adc - 1), + }) + hasher.Write(buf[12:bodyend]) + + hashed := hasher.Sum(nil) + sig := buf[sigend:] + switch k.Algorithm { + case DSA: + pk := k.publicKeyDSA() + sig = sig[1:] + r := big.NewInt(0) + r.SetBytes(sig[:len(sig)/2]) + s := big.NewInt(0) + s.SetBytes(sig[len(sig)/2:]) + if pk != nil { + if dsa.Verify(pk, hashed, r, s) { + return nil + } + return ErrSig + } + case RSASHA1, RSASHA256, RSASHA512: + pk := k.publicKeyRSA() + if pk != nil { + return rsa.VerifyPKCS1v15(pk, hash, hashed, sig) + } + case ECDSAP256SHA256, ECDSAP384SHA384: + pk := k.publicKeyECDSA() + r := big.NewInt(0) + r.SetBytes(sig[:len(sig)/2]) + s := big.NewInt(0) + s.SetBytes(sig[len(sig)/2:]) + if pk != nil { + if ecdsa.Verify(pk, hashed, r, s) { + return nil + } + return ErrSig + } + } + return ErrKeyAlg +} diff --git a/vendor/github.com/miekg/dns/singleinflight.go b/vendor/github.com/miekg/dns/singleinflight.go new file mode 100644 index 00000000..9573c7d0 --- /dev/null +++ b/vendor/github.com/miekg/dns/singleinflight.go @@ -0,0 +1,57 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Adapted for dns package usage by Miek Gieben. + +package dns + +import "sync" +import "time" + +// call is an in-flight or completed singleflight.Do call +type call struct { + wg sync.WaitGroup + val *Msg + rtt time.Duration + err error + dups int +} + +// singleflight represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type singleflight struct { + sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *singleflight) Do(key string, fn func() (*Msg, time.Duration, error)) (v *Msg, rtt time.Duration, err error, shared bool) { + g.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.Unlock() + c.wg.Wait() + return c.val, c.rtt, c.err, true + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.Unlock() + + c.val, c.rtt, c.err = fn() + c.wg.Done() + + g.Lock() + delete(g.m, key) + g.Unlock() + + return c.val, c.rtt, c.err, c.dups > 0 +} diff --git a/vendor/github.com/miekg/dns/smimea.go b/vendor/github.com/miekg/dns/smimea.go new file mode 100644 index 00000000..4e7ded4b --- /dev/null +++ b/vendor/github.com/miekg/dns/smimea.go @@ -0,0 +1,47 @@ +package dns + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/hex" +) + +// Sign creates a SMIMEA record from an SSL certificate. +func (r *SMIMEA) Sign(usage, selector, matchingType int, cert *x509.Certificate) (err error) { + r.Hdr.Rrtype = TypeSMIMEA + r.Usage = uint8(usage) + r.Selector = uint8(selector) + r.MatchingType = uint8(matchingType) + + r.Certificate, err = CertificateToDANE(r.Selector, r.MatchingType, cert) + if err != nil { + return err + } + return nil +} + +// Verify verifies a SMIMEA record against an SSL certificate. If it is OK +// a nil error is returned. +func (r *SMIMEA) Verify(cert *x509.Certificate) error { + c, err := CertificateToDANE(r.Selector, r.MatchingType, cert) + if err != nil { + return err // Not also ErrSig? + } + if r.Certificate == c { + return nil + } + return ErrSig // ErrSig, really? +} + +// SMIMEAName returns the ownername of a SMIMEA resource record as per the +// format specified in RFC 'draft-ietf-dane-smime-12' Section 2 and 3 +func SMIMEAName(email, domain string) (string, error) { + hasher := sha256.New() + hasher.Write([]byte(email)) + + // RFC Section 3: "The local-part is hashed using the SHA2-256 + // algorithm with the hash truncated to 28 octets and + // represented in its hexadecimal representation to become the + // left-most label in the prepared domain name" + return hex.EncodeToString(hasher.Sum(nil)[:28]) + "." + "_smimecert." + domain, nil +} diff --git a/vendor/github.com/miekg/dns/tlsa.go b/vendor/github.com/miekg/dns/tlsa.go new file mode 100644 index 00000000..431e2fb5 --- /dev/null +++ b/vendor/github.com/miekg/dns/tlsa.go @@ -0,0 +1,47 @@ +package dns + +import ( + "crypto/x509" + "net" + "strconv" +) + +// Sign creates a TLSA record from an SSL certificate. +func (r *TLSA) Sign(usage, selector, matchingType int, cert *x509.Certificate) (err error) { + r.Hdr.Rrtype = TypeTLSA + r.Usage = uint8(usage) + r.Selector = uint8(selector) + r.MatchingType = uint8(matchingType) + + r.Certificate, err = CertificateToDANE(r.Selector, r.MatchingType, cert) + if err != nil { + return err + } + return nil +} + +// Verify verifies a TLSA record against an SSL certificate. If it is OK +// a nil error is returned. +func (r *TLSA) Verify(cert *x509.Certificate) error { + c, err := CertificateToDANE(r.Selector, r.MatchingType, cert) + if err != nil { + return err // Not also ErrSig? + } + if r.Certificate == c { + return nil + } + return ErrSig // ErrSig, really? +} + +// TLSAName returns the ownername of a TLSA resource record as per the +// rules specified in RFC 6698, Section 3. +func TLSAName(name, service, network string) (string, error) { + if !IsFqdn(name) { + return "", ErrFqdn + } + p, err := net.LookupPort(network, service) + if err != nil { + return "", err + } + return "_" + strconv.Itoa(p) + "._" + network + "." + name, nil +} diff --git a/vendor/github.com/miekg/dns/tsig.go b/vendor/github.com/miekg/dns/tsig.go new file mode 100644 index 00000000..24013096 --- /dev/null +++ b/vendor/github.com/miekg/dns/tsig.go @@ -0,0 +1,383 @@ +package dns + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/binary" + "encoding/hex" + "hash" + "strconv" + "strings" + "time" +) + +// HMAC hashing codes. These are transmitted as domain names. +const ( + HmacMD5 = "hmac-md5.sig-alg.reg.int." + HmacSHA1 = "hmac-sha1." + HmacSHA256 = "hmac-sha256." + HmacSHA512 = "hmac-sha512." +) + +// TSIG is the RR the holds the transaction signature of a message. +// See RFC 2845 and RFC 4635. +type TSIG struct { + Hdr RR_Header + Algorithm string `dns:"domain-name"` + TimeSigned uint64 `dns:"uint48"` + Fudge uint16 + MACSize uint16 + MAC string `dns:"size-hex:MACSize"` + OrigId uint16 + Error uint16 + OtherLen uint16 + OtherData string `dns:"size-hex:OtherLen"` +} + +// TSIG has no official presentation format, but this will suffice. + +func (rr *TSIG) String() string { + s := "\n;; TSIG PSEUDOSECTION:\n" + s += rr.Hdr.String() + + " " + rr.Algorithm + + " " + tsigTimeToString(rr.TimeSigned) + + " " + strconv.Itoa(int(rr.Fudge)) + + " " + strconv.Itoa(int(rr.MACSize)) + + " " + strings.ToUpper(rr.MAC) + + " " + strconv.Itoa(int(rr.OrigId)) + + " " + strconv.Itoa(int(rr.Error)) + // BIND prints NOERROR + " " + strconv.Itoa(int(rr.OtherLen)) + + " " + rr.OtherData + return s +} + +// The following values must be put in wireformat, so that the MAC can be calculated. +// RFC 2845, section 3.4.2. TSIG Variables. +type tsigWireFmt struct { + // From RR_Header + Name string `dns:"domain-name"` + Class uint16 + Ttl uint32 + // Rdata of the TSIG + Algorithm string `dns:"domain-name"` + TimeSigned uint64 `dns:"uint48"` + Fudge uint16 + // MACSize, MAC and OrigId excluded + Error uint16 + OtherLen uint16 + OtherData string `dns:"size-hex:OtherLen"` +} + +// If we have the MAC use this type to convert it to wiredata. Section 3.4.3. Request MAC +type macWireFmt struct { + MACSize uint16 + MAC string `dns:"size-hex:MACSize"` +} + +// 3.3. Time values used in TSIG calculations +type timerWireFmt struct { + TimeSigned uint64 `dns:"uint48"` + Fudge uint16 +} + +// TsigGenerate fills out the TSIG record attached to the message. +// The message should contain +// a "stub" TSIG RR with the algorithm, key name (owner name of the RR), +// time fudge (defaults to 300 seconds) and the current time +// The TSIG MAC is saved in that Tsig RR. +// When TsigGenerate is called for the first time requestMAC is set to the empty string and +// timersOnly is false. +// If something goes wrong an error is returned, otherwise it is nil. +func TsigGenerate(m *Msg, secret, requestMAC string, timersOnly bool) ([]byte, string, error) { + if m.IsTsig() == nil { + panic("dns: TSIG not last RR in additional") + } + // If we barf here, the caller is to blame + rawsecret, err := fromBase64([]byte(secret)) + if err != nil { + return nil, "", err + } + + rr := m.Extra[len(m.Extra)-1].(*TSIG) + m.Extra = m.Extra[0 : len(m.Extra)-1] // kill the TSIG from the msg + mbuf, err := m.Pack() + if err != nil { + return nil, "", err + } + buf := tsigBuffer(mbuf, rr, requestMAC, timersOnly) + + t := new(TSIG) + var h hash.Hash + switch strings.ToLower(rr.Algorithm) { + case HmacMD5: + h = hmac.New(md5.New, []byte(rawsecret)) + case HmacSHA1: + h = hmac.New(sha1.New, []byte(rawsecret)) + case HmacSHA256: + h = hmac.New(sha256.New, []byte(rawsecret)) + case HmacSHA512: + h = hmac.New(sha512.New, []byte(rawsecret)) + default: + return nil, "", ErrKeyAlg + } + h.Write(buf) + t.MAC = hex.EncodeToString(h.Sum(nil)) + t.MACSize = uint16(len(t.MAC) / 2) // Size is half! + + t.Hdr = RR_Header{Name: rr.Hdr.Name, Rrtype: TypeTSIG, Class: ClassANY, Ttl: 0} + t.Fudge = rr.Fudge + t.TimeSigned = rr.TimeSigned + t.Algorithm = rr.Algorithm + t.OrigId = m.Id + + tbuf := make([]byte, t.len()) + if off, err := PackRR(t, tbuf, 0, nil, false); err == nil { + tbuf = tbuf[:off] // reset to actual size used + } else { + return nil, "", err + } + mbuf = append(mbuf, tbuf...) + // Update the ArCount directly in the buffer. + binary.BigEndian.PutUint16(mbuf[10:], uint16(len(m.Extra)+1)) + + return mbuf, t.MAC, nil +} + +// TsigVerify verifies the TSIG on a message. +// If the signature does not validate err contains the +// error, otherwise it is nil. +func TsigVerify(msg []byte, secret, requestMAC string, timersOnly bool) error { + rawsecret, err := fromBase64([]byte(secret)) + if err != nil { + return err + } + // Strip the TSIG from the incoming msg + stripped, tsig, err := stripTsig(msg) + if err != nil { + return err + } + + msgMAC, err := hex.DecodeString(tsig.MAC) + if err != nil { + return err + } + + buf := tsigBuffer(stripped, tsig, requestMAC, timersOnly) + + // Fudge factor works both ways. A message can arrive before it was signed because + // of clock skew. + now := uint64(time.Now().Unix()) + ti := now - tsig.TimeSigned + if now < tsig.TimeSigned { + ti = tsig.TimeSigned - now + } + if uint64(tsig.Fudge) < ti { + return ErrTime + } + + var h hash.Hash + switch strings.ToLower(tsig.Algorithm) { + case HmacMD5: + h = hmac.New(md5.New, rawsecret) + case HmacSHA1: + h = hmac.New(sha1.New, rawsecret) + case HmacSHA256: + h = hmac.New(sha256.New, rawsecret) + case HmacSHA512: + h = hmac.New(sha512.New, rawsecret) + default: + return ErrKeyAlg + } + h.Write(buf) + if !hmac.Equal(h.Sum(nil), msgMAC) { + return ErrSig + } + return nil +} + +// Create a wiredata buffer for the MAC calculation. +func tsigBuffer(msgbuf []byte, rr *TSIG, requestMAC string, timersOnly bool) []byte { + var buf []byte + if rr.TimeSigned == 0 { + rr.TimeSigned = uint64(time.Now().Unix()) + } + if rr.Fudge == 0 { + rr.Fudge = 300 // Standard (RFC) default. + } + + if requestMAC != "" { + m := new(macWireFmt) + m.MACSize = uint16(len(requestMAC) / 2) + m.MAC = requestMAC + buf = make([]byte, len(requestMAC)) // long enough + n, _ := packMacWire(m, buf) + buf = buf[:n] + } + + tsigvar := make([]byte, DefaultMsgSize) + if timersOnly { + tsig := new(timerWireFmt) + tsig.TimeSigned = rr.TimeSigned + tsig.Fudge = rr.Fudge + n, _ := packTimerWire(tsig, tsigvar) + tsigvar = tsigvar[:n] + } else { + tsig := new(tsigWireFmt) + tsig.Name = strings.ToLower(rr.Hdr.Name) + tsig.Class = ClassANY + tsig.Ttl = rr.Hdr.Ttl + tsig.Algorithm = strings.ToLower(rr.Algorithm) + tsig.TimeSigned = rr.TimeSigned + tsig.Fudge = rr.Fudge + tsig.Error = rr.Error + tsig.OtherLen = rr.OtherLen + tsig.OtherData = rr.OtherData + n, _ := packTsigWire(tsig, tsigvar) + tsigvar = tsigvar[:n] + } + + if requestMAC != "" { + x := append(buf, msgbuf...) + buf = append(x, tsigvar...) + } else { + buf = append(msgbuf, tsigvar...) + } + return buf +} + +// Strip the TSIG from the raw message. +func stripTsig(msg []byte) ([]byte, *TSIG, error) { + // Copied from msg.go's Unpack() Header, but modified. + var ( + dh Header + err error + ) + off, tsigoff := 0, 0 + + if dh, off, err = unpackMsgHdr(msg, off); err != nil { + return nil, nil, err + } + if dh.Arcount == 0 { + return nil, nil, ErrNoSig + } + + // Rcode, see msg.go Unpack() + if int(dh.Bits&0xF) == RcodeNotAuth { + return nil, nil, ErrAuth + } + + for i := 0; i < int(dh.Qdcount); i++ { + _, off, err = unpackQuestion(msg, off) + if err != nil { + return nil, nil, err + } + } + + _, off, err = unpackRRslice(int(dh.Ancount), msg, off) + if err != nil { + return nil, nil, err + } + _, off, err = unpackRRslice(int(dh.Nscount), msg, off) + if err != nil { + return nil, nil, err + } + + rr := new(TSIG) + var extra RR + for i := 0; i < int(dh.Arcount); i++ { + tsigoff = off + extra, off, err = UnpackRR(msg, off) + if err != nil { + return nil, nil, err + } + if extra.Header().Rrtype == TypeTSIG { + rr = extra.(*TSIG) + // Adjust Arcount. + arcount := binary.BigEndian.Uint16(msg[10:]) + binary.BigEndian.PutUint16(msg[10:], arcount-1) + break + } + } + if rr == nil { + return nil, nil, ErrNoSig + } + return msg[:tsigoff], rr, nil +} + +// Translate the TSIG time signed into a date. There is no +// need for RFC1982 calculations as this date is 48 bits. +func tsigTimeToString(t uint64) string { + ti := time.Unix(int64(t), 0).UTC() + return ti.Format("20060102150405") +} + +func packTsigWire(tw *tsigWireFmt, msg []byte) (int, error) { + // copied from zmsg.go TSIG packing + // RR_Header + off, err := PackDomainName(tw.Name, msg, 0, nil, false) + if err != nil { + return off, err + } + off, err = packUint16(tw.Class, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(tw.Ttl, msg, off) + if err != nil { + return off, err + } + + off, err = PackDomainName(tw.Algorithm, msg, off, nil, false) + if err != nil { + return off, err + } + off, err = packUint48(tw.TimeSigned, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(tw.Fudge, msg, off) + if err != nil { + return off, err + } + + off, err = packUint16(tw.Error, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(tw.OtherLen, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(tw.OtherData, msg, off) + if err != nil { + return off, err + } + return off, nil +} + +func packMacWire(mw *macWireFmt, msg []byte) (int, error) { + off, err := packUint16(mw.MACSize, msg, 0) + if err != nil { + return off, err + } + off, err = packStringHex(mw.MAC, msg, off) + if err != nil { + return off, err + } + return off, nil +} + +func packTimerWire(tw *timerWireFmt, msg []byte) (int, error) { + off, err := packUint48(tw.TimeSigned, msg, 0) + if err != nil { + return off, err + } + off, err = packUint16(tw.Fudge, msg, off) + if err != nil { + return off, err + } + return off, nil +} diff --git a/vendor/github.com/miekg/dns/types.go b/vendor/github.com/miekg/dns/types.go new file mode 100644 index 00000000..53da4755 --- /dev/null +++ b/vendor/github.com/miekg/dns/types.go @@ -0,0 +1,1287 @@ +package dns + +import ( + "fmt" + "net" + "strconv" + "strings" + "time" +) + +type ( + // Type is a DNS type. + Type uint16 + // Class is a DNS class. + Class uint16 + // Name is a DNS domain name. + Name string +) + +// Packet formats + +// Wire constants and supported types. +const ( + // valid RR_Header.Rrtype and Question.qtype + + TypeNone uint16 = 0 + TypeA uint16 = 1 + TypeNS uint16 = 2 + TypeMD uint16 = 3 + TypeMF uint16 = 4 + TypeCNAME uint16 = 5 + TypeSOA uint16 = 6 + TypeMB uint16 = 7 + TypeMG uint16 = 8 + TypeMR uint16 = 9 + TypeNULL uint16 = 10 + TypePTR uint16 = 12 + TypeHINFO uint16 = 13 + TypeMINFO uint16 = 14 + TypeMX uint16 = 15 + TypeTXT uint16 = 16 + TypeRP uint16 = 17 + TypeAFSDB uint16 = 18 + TypeX25 uint16 = 19 + TypeISDN uint16 = 20 + TypeRT uint16 = 21 + TypeNSAPPTR uint16 = 23 + TypeSIG uint16 = 24 + TypeKEY uint16 = 25 + TypePX uint16 = 26 + TypeGPOS uint16 = 27 + TypeAAAA uint16 = 28 + TypeLOC uint16 = 29 + TypeNXT uint16 = 30 + TypeEID uint16 = 31 + TypeNIMLOC uint16 = 32 + TypeSRV uint16 = 33 + TypeATMA uint16 = 34 + TypeNAPTR uint16 = 35 + TypeKX uint16 = 36 + TypeCERT uint16 = 37 + TypeDNAME uint16 = 39 + TypeOPT uint16 = 41 // EDNS + TypeDS uint16 = 43 + TypeSSHFP uint16 = 44 + TypeRRSIG uint16 = 46 + TypeNSEC uint16 = 47 + TypeDNSKEY uint16 = 48 + TypeDHCID uint16 = 49 + TypeNSEC3 uint16 = 50 + TypeNSEC3PARAM uint16 = 51 + TypeTLSA uint16 = 52 + TypeSMIMEA uint16 = 53 + TypeHIP uint16 = 55 + TypeNINFO uint16 = 56 + TypeRKEY uint16 = 57 + TypeTALINK uint16 = 58 + TypeCDS uint16 = 59 + TypeCDNSKEY uint16 = 60 + TypeOPENPGPKEY uint16 = 61 + TypeSPF uint16 = 99 + TypeUINFO uint16 = 100 + TypeUID uint16 = 101 + TypeGID uint16 = 102 + TypeUNSPEC uint16 = 103 + TypeNID uint16 = 104 + TypeL32 uint16 = 105 + TypeL64 uint16 = 106 + TypeLP uint16 = 107 + TypeEUI48 uint16 = 108 + TypeEUI64 uint16 = 109 + TypeURI uint16 = 256 + TypeCAA uint16 = 257 + TypeAVC uint16 = 258 + + TypeTKEY uint16 = 249 + TypeTSIG uint16 = 250 + + // valid Question.Qtype only + TypeIXFR uint16 = 251 + TypeAXFR uint16 = 252 + TypeMAILB uint16 = 253 + TypeMAILA uint16 = 254 + TypeANY uint16 = 255 + + TypeTA uint16 = 32768 + TypeDLV uint16 = 32769 + TypeReserved uint16 = 65535 + + // valid Question.Qclass + ClassINET = 1 + ClassCSNET = 2 + ClassCHAOS = 3 + ClassHESIOD = 4 + ClassNONE = 254 + ClassANY = 255 + + // Message Response Codes. + RcodeSuccess = 0 + RcodeFormatError = 1 + RcodeServerFailure = 2 + RcodeNameError = 3 + RcodeNotImplemented = 4 + RcodeRefused = 5 + RcodeYXDomain = 6 + RcodeYXRrset = 7 + RcodeNXRrset = 8 + RcodeNotAuth = 9 + RcodeNotZone = 10 + RcodeBadSig = 16 // TSIG + RcodeBadVers = 16 // EDNS0 + RcodeBadKey = 17 + RcodeBadTime = 18 + RcodeBadMode = 19 // TKEY + RcodeBadName = 20 + RcodeBadAlg = 21 + RcodeBadTrunc = 22 // TSIG + RcodeBadCookie = 23 // DNS Cookies + + // Message Opcodes. There is no 3. + OpcodeQuery = 0 + OpcodeIQuery = 1 + OpcodeStatus = 2 + OpcodeNotify = 4 + OpcodeUpdate = 5 +) + +// Header is the wire format for the DNS packet header. +type Header struct { + Id uint16 + Bits uint16 + Qdcount, Ancount, Nscount, Arcount uint16 +} + +const ( + headerSize = 12 + + // Header.Bits + _QR = 1 << 15 // query/response (response=1) + _AA = 1 << 10 // authoritative + _TC = 1 << 9 // truncated + _RD = 1 << 8 // recursion desired + _RA = 1 << 7 // recursion available + _Z = 1 << 6 // Z + _AD = 1 << 5 // authticated data + _CD = 1 << 4 // checking disabled + + LOC_EQUATOR = 1 << 31 // RFC 1876, Section 2. + LOC_PRIMEMERIDIAN = 1 << 31 // RFC 1876, Section 2. + + LOC_HOURS = 60 * 1000 + LOC_DEGREES = 60 * LOC_HOURS + + LOC_ALTITUDEBASE = 100000 +) + +// Different Certificate Types, see RFC 4398, Section 2.1 +const ( + CertPKIX = 1 + iota + CertSPKI + CertPGP + CertIPIX + CertISPKI + CertIPGP + CertACPKIX + CertIACPKIX + CertURI = 253 + CertOID = 254 +) + +// CertTypeToString converts the Cert Type to its string representation. +// See RFC 4398 and RFC 6944. +var CertTypeToString = map[uint16]string{ + CertPKIX: "PKIX", + CertSPKI: "SPKI", + CertPGP: "PGP", + CertIPIX: "IPIX", + CertISPKI: "ISPKI", + CertIPGP: "IPGP", + CertACPKIX: "ACPKIX", + CertIACPKIX: "IACPKIX", + CertURI: "URI", + CertOID: "OID", +} + +// StringToCertType is the reverseof CertTypeToString. +var StringToCertType = reverseInt16(CertTypeToString) + +//go:generate go run types_generate.go + +// Question holds a DNS question. There can be multiple questions in the +// question section of a message. Usually there is just one. +type Question struct { + Name string `dns:"cdomain-name"` // "cdomain-name" specifies encoding (and may be compressed) + Qtype uint16 + Qclass uint16 +} + +func (q *Question) len() int { + return len(q.Name) + 1 + 2 + 2 +} + +func (q *Question) String() (s string) { + // prefix with ; (as in dig) + s = ";" + sprintName(q.Name) + "\t" + s += Class(q.Qclass).String() + "\t" + s += " " + Type(q.Qtype).String() + return s +} + +// ANY is a wildcard record. See RFC 1035, Section 3.2.3. ANY +// is named "*" there. +type ANY struct { + Hdr RR_Header + // Does not have any rdata +} + +func (rr *ANY) String() string { return rr.Hdr.String() } + +type CNAME struct { + Hdr RR_Header + Target string `dns:"cdomain-name"` +} + +func (rr *CNAME) String() string { return rr.Hdr.String() + sprintName(rr.Target) } + +type HINFO struct { + Hdr RR_Header + Cpu string + Os string +} + +func (rr *HINFO) String() string { + return rr.Hdr.String() + sprintTxt([]string{rr.Cpu, rr.Os}) +} + +type MB struct { + Hdr RR_Header + Mb string `dns:"cdomain-name"` +} + +func (rr *MB) String() string { return rr.Hdr.String() + sprintName(rr.Mb) } + +type MG struct { + Hdr RR_Header + Mg string `dns:"cdomain-name"` +} + +func (rr *MG) String() string { return rr.Hdr.String() + sprintName(rr.Mg) } + +type MINFO struct { + Hdr RR_Header + Rmail string `dns:"cdomain-name"` + Email string `dns:"cdomain-name"` +} + +func (rr *MINFO) String() string { + return rr.Hdr.String() + sprintName(rr.Rmail) + " " + sprintName(rr.Email) +} + +type MR struct { + Hdr RR_Header + Mr string `dns:"cdomain-name"` +} + +func (rr *MR) String() string { + return rr.Hdr.String() + sprintName(rr.Mr) +} + +type MF struct { + Hdr RR_Header + Mf string `dns:"cdomain-name"` +} + +func (rr *MF) String() string { + return rr.Hdr.String() + sprintName(rr.Mf) +} + +type MD struct { + Hdr RR_Header + Md string `dns:"cdomain-name"` +} + +func (rr *MD) String() string { + return rr.Hdr.String() + sprintName(rr.Md) +} + +type MX struct { + Hdr RR_Header + Preference uint16 + Mx string `dns:"cdomain-name"` +} + +func (rr *MX) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Mx) +} + +type AFSDB struct { + Hdr RR_Header + Subtype uint16 + Hostname string `dns:"cdomain-name"` +} + +func (rr *AFSDB) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Subtype)) + " " + sprintName(rr.Hostname) +} + +type X25 struct { + Hdr RR_Header + PSDNAddress string +} + +func (rr *X25) String() string { + return rr.Hdr.String() + rr.PSDNAddress +} + +type RT struct { + Hdr RR_Header + Preference uint16 + Host string `dns:"cdomain-name"` +} + +func (rr *RT) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Host) +} + +type NS struct { + Hdr RR_Header + Ns string `dns:"cdomain-name"` +} + +func (rr *NS) String() string { + return rr.Hdr.String() + sprintName(rr.Ns) +} + +type PTR struct { + Hdr RR_Header + Ptr string `dns:"cdomain-name"` +} + +func (rr *PTR) String() string { + return rr.Hdr.String() + sprintName(rr.Ptr) +} + +type RP struct { + Hdr RR_Header + Mbox string `dns:"domain-name"` + Txt string `dns:"domain-name"` +} + +func (rr *RP) String() string { + return rr.Hdr.String() + rr.Mbox + " " + sprintTxt([]string{rr.Txt}) +} + +type SOA struct { + Hdr RR_Header + Ns string `dns:"cdomain-name"` + Mbox string `dns:"cdomain-name"` + Serial uint32 + Refresh uint32 + Retry uint32 + Expire uint32 + Minttl uint32 +} + +func (rr *SOA) String() string { + return rr.Hdr.String() + sprintName(rr.Ns) + " " + sprintName(rr.Mbox) + + " " + strconv.FormatInt(int64(rr.Serial), 10) + + " " + strconv.FormatInt(int64(rr.Refresh), 10) + + " " + strconv.FormatInt(int64(rr.Retry), 10) + + " " + strconv.FormatInt(int64(rr.Expire), 10) + + " " + strconv.FormatInt(int64(rr.Minttl), 10) +} + +type TXT struct { + Hdr RR_Header + Txt []string `dns:"txt"` +} + +func (rr *TXT) String() string { return rr.Hdr.String() + sprintTxt(rr.Txt) } + +func sprintName(s string) string { + src := []byte(s) + dst := make([]byte, 0, len(src)) + for i := 0; i < len(src); { + if i+1 < len(src) && src[i] == '\\' && src[i+1] == '.' { + dst = append(dst, src[i:i+2]...) + i += 2 + } else { + b, n := nextByte(src, i) + if n == 0 { + i++ // dangling back slash + } else if b == '.' { + dst = append(dst, b) + } else { + dst = appendDomainNameByte(dst, b) + } + i += n + } + } + return string(dst) +} + +func sprintTxtOctet(s string) string { + src := []byte(s) + dst := make([]byte, 0, len(src)) + dst = append(dst, '"') + for i := 0; i < len(src); { + if i+1 < len(src) && src[i] == '\\' && src[i+1] == '.' { + dst = append(dst, src[i:i+2]...) + i += 2 + } else { + b, n := nextByte(src, i) + if n == 0 { + i++ // dangling back slash + } else if b == '.' { + dst = append(dst, b) + } else { + if b < ' ' || b > '~' { + dst = appendByte(dst, b) + } else { + dst = append(dst, b) + } + } + i += n + } + } + dst = append(dst, '"') + return string(dst) +} + +func sprintTxt(txt []string) string { + var out []byte + for i, s := range txt { + if i > 0 { + out = append(out, ` "`...) + } else { + out = append(out, '"') + } + bs := []byte(s) + for j := 0; j < len(bs); { + b, n := nextByte(bs, j) + if n == 0 { + break + } + out = appendTXTStringByte(out, b) + j += n + } + out = append(out, '"') + } + return string(out) +} + +func appendDomainNameByte(s []byte, b byte) []byte { + switch b { + case '.', ' ', '\'', '@', ';', '(', ')': // additional chars to escape + return append(s, '\\', b) + } + return appendTXTStringByte(s, b) +} + +func appendTXTStringByte(s []byte, b byte) []byte { + switch b { + case '"', '\\': + return append(s, '\\', b) + } + if b < ' ' || b > '~' { + return appendByte(s, b) + } + return append(s, b) +} + +func appendByte(s []byte, b byte) []byte { + var buf [3]byte + bufs := strconv.AppendInt(buf[:0], int64(b), 10) + s = append(s, '\\') + for i := 0; i < 3-len(bufs); i++ { + s = append(s, '0') + } + for _, r := range bufs { + s = append(s, r) + } + return s +} + +func nextByte(b []byte, offset int) (byte, int) { + if offset >= len(b) { + return 0, 0 + } + if b[offset] != '\\' { + // not an escape sequence + return b[offset], 1 + } + switch len(b) - offset { + case 1: // dangling escape + return 0, 0 + case 2, 3: // too short to be \ddd + default: // maybe \ddd + if isDigit(b[offset+1]) && isDigit(b[offset+2]) && isDigit(b[offset+3]) { + return dddToByte(b[offset+1:]), 4 + } + } + // not \ddd, just an RFC 1035 "quoted" character + return b[offset+1], 2 +} + +type SPF struct { + Hdr RR_Header + Txt []string `dns:"txt"` +} + +func (rr *SPF) String() string { return rr.Hdr.String() + sprintTxt(rr.Txt) } + +type AVC struct { + Hdr RR_Header + Txt []string `dns:"txt"` +} + +func (rr *AVC) String() string { return rr.Hdr.String() + sprintTxt(rr.Txt) } + +type SRV struct { + Hdr RR_Header + Priority uint16 + Weight uint16 + Port uint16 + Target string `dns:"domain-name"` +} + +func (rr *SRV) String() string { + return rr.Hdr.String() + + strconv.Itoa(int(rr.Priority)) + " " + + strconv.Itoa(int(rr.Weight)) + " " + + strconv.Itoa(int(rr.Port)) + " " + sprintName(rr.Target) +} + +type NAPTR struct { + Hdr RR_Header + Order uint16 + Preference uint16 + Flags string + Service string + Regexp string + Replacement string `dns:"domain-name"` +} + +func (rr *NAPTR) String() string { + return rr.Hdr.String() + + strconv.Itoa(int(rr.Order)) + " " + + strconv.Itoa(int(rr.Preference)) + " " + + "\"" + rr.Flags + "\" " + + "\"" + rr.Service + "\" " + + "\"" + rr.Regexp + "\" " + + rr.Replacement +} + +// The CERT resource record, see RFC 4398. +type CERT struct { + Hdr RR_Header + Type uint16 + KeyTag uint16 + Algorithm uint8 + Certificate string `dns:"base64"` +} + +func (rr *CERT) String() string { + var ( + ok bool + certtype, algorithm string + ) + if certtype, ok = CertTypeToString[rr.Type]; !ok { + certtype = strconv.Itoa(int(rr.Type)) + } + if algorithm, ok = AlgorithmToString[rr.Algorithm]; !ok { + algorithm = strconv.Itoa(int(rr.Algorithm)) + } + return rr.Hdr.String() + certtype + + " " + strconv.Itoa(int(rr.KeyTag)) + + " " + algorithm + + " " + rr.Certificate +} + +// The DNAME resource record, see RFC 2672. +type DNAME struct { + Hdr RR_Header + Target string `dns:"domain-name"` +} + +func (rr *DNAME) String() string { + return rr.Hdr.String() + sprintName(rr.Target) +} + +type A struct { + Hdr RR_Header + A net.IP `dns:"a"` +} + +func (rr *A) String() string { + if rr.A == nil { + return rr.Hdr.String() + } + return rr.Hdr.String() + rr.A.String() +} + +type AAAA struct { + Hdr RR_Header + AAAA net.IP `dns:"aaaa"` +} + +func (rr *AAAA) String() string { + if rr.AAAA == nil { + return rr.Hdr.String() + } + return rr.Hdr.String() + rr.AAAA.String() +} + +type PX struct { + Hdr RR_Header + Preference uint16 + Map822 string `dns:"domain-name"` + Mapx400 string `dns:"domain-name"` +} + +func (rr *PX) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Map822) + " " + sprintName(rr.Mapx400) +} + +type GPOS struct { + Hdr RR_Header + Longitude string + Latitude string + Altitude string +} + +func (rr *GPOS) String() string { + return rr.Hdr.String() + rr.Longitude + " " + rr.Latitude + " " + rr.Altitude +} + +type LOC struct { + Hdr RR_Header + Version uint8 + Size uint8 + HorizPre uint8 + VertPre uint8 + Latitude uint32 + Longitude uint32 + Altitude uint32 +} + +// cmToM takes a cm value expressed in RFC1876 SIZE mantissa/exponent +// format and returns a string in m (two decimals for the cm) +func cmToM(m, e uint8) string { + if e < 2 { + if e == 1 { + m *= 10 + } + + return fmt.Sprintf("0.%02d", m) + } + + s := fmt.Sprintf("%d", m) + for e > 2 { + s += "0" + e-- + } + return s +} + +func (rr *LOC) String() string { + s := rr.Hdr.String() + + lat := rr.Latitude + ns := "N" + if lat > LOC_EQUATOR { + lat = lat - LOC_EQUATOR + } else { + ns = "S" + lat = LOC_EQUATOR - lat + } + h := lat / LOC_DEGREES + lat = lat % LOC_DEGREES + m := lat / LOC_HOURS + lat = lat % LOC_HOURS + s += fmt.Sprintf("%02d %02d %0.3f %s ", h, m, (float64(lat) / 1000), ns) + + lon := rr.Longitude + ew := "E" + if lon > LOC_PRIMEMERIDIAN { + lon = lon - LOC_PRIMEMERIDIAN + } else { + ew = "W" + lon = LOC_PRIMEMERIDIAN - lon + } + h = lon / LOC_DEGREES + lon = lon % LOC_DEGREES + m = lon / LOC_HOURS + lon = lon % LOC_HOURS + s += fmt.Sprintf("%02d %02d %0.3f %s ", h, m, (float64(lon) / 1000), ew) + + var alt = float64(rr.Altitude) / 100 + alt -= LOC_ALTITUDEBASE + if rr.Altitude%100 != 0 { + s += fmt.Sprintf("%.2fm ", alt) + } else { + s += fmt.Sprintf("%.0fm ", alt) + } + + s += cmToM((rr.Size&0xf0)>>4, rr.Size&0x0f) + "m " + s += cmToM((rr.HorizPre&0xf0)>>4, rr.HorizPre&0x0f) + "m " + s += cmToM((rr.VertPre&0xf0)>>4, rr.VertPre&0x0f) + "m" + + return s +} + +// SIG is identical to RRSIG and nowadays only used for SIG(0), RFC2931. +type SIG struct { + RRSIG +} + +type RRSIG struct { + Hdr RR_Header + TypeCovered uint16 + Algorithm uint8 + Labels uint8 + OrigTtl uint32 + Expiration uint32 + Inception uint32 + KeyTag uint16 + SignerName string `dns:"domain-name"` + Signature string `dns:"base64"` +} + +func (rr *RRSIG) String() string { + s := rr.Hdr.String() + s += Type(rr.TypeCovered).String() + s += " " + strconv.Itoa(int(rr.Algorithm)) + + " " + strconv.Itoa(int(rr.Labels)) + + " " + strconv.FormatInt(int64(rr.OrigTtl), 10) + + " " + TimeToString(rr.Expiration) + + " " + TimeToString(rr.Inception) + + " " + strconv.Itoa(int(rr.KeyTag)) + + " " + sprintName(rr.SignerName) + + " " + rr.Signature + return s +} + +type NSEC struct { + Hdr RR_Header + NextDomain string `dns:"domain-name"` + TypeBitMap []uint16 `dns:"nsec"` +} + +func (rr *NSEC) String() string { + s := rr.Hdr.String() + sprintName(rr.NextDomain) + for i := 0; i < len(rr.TypeBitMap); i++ { + s += " " + Type(rr.TypeBitMap[i]).String() + } + return s +} + +func (rr *NSEC) len() int { + l := rr.Hdr.len() + len(rr.NextDomain) + 1 + lastwindow := uint32(2 ^ 32 + 1) + for _, t := range rr.TypeBitMap { + window := t / 256 + if uint32(window) != lastwindow { + l += 1 + 32 + } + lastwindow = uint32(window) + } + return l +} + +type DLV struct { + DS +} + +type CDS struct { + DS +} + +type DS struct { + Hdr RR_Header + KeyTag uint16 + Algorithm uint8 + DigestType uint8 + Digest string `dns:"hex"` +} + +func (rr *DS) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.KeyTag)) + + " " + strconv.Itoa(int(rr.Algorithm)) + + " " + strconv.Itoa(int(rr.DigestType)) + + " " + strings.ToUpper(rr.Digest) +} + +type KX struct { + Hdr RR_Header + Preference uint16 + Exchanger string `dns:"domain-name"` +} + +func (rr *KX) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + + " " + sprintName(rr.Exchanger) +} + +type TA struct { + Hdr RR_Header + KeyTag uint16 + Algorithm uint8 + DigestType uint8 + Digest string `dns:"hex"` +} + +func (rr *TA) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.KeyTag)) + + " " + strconv.Itoa(int(rr.Algorithm)) + + " " + strconv.Itoa(int(rr.DigestType)) + + " " + strings.ToUpper(rr.Digest) +} + +type TALINK struct { + Hdr RR_Header + PreviousName string `dns:"domain-name"` + NextName string `dns:"domain-name"` +} + +func (rr *TALINK) String() string { + return rr.Hdr.String() + + sprintName(rr.PreviousName) + " " + sprintName(rr.NextName) +} + +type SSHFP struct { + Hdr RR_Header + Algorithm uint8 + Type uint8 + FingerPrint string `dns:"hex"` +} + +func (rr *SSHFP) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Algorithm)) + + " " + strconv.Itoa(int(rr.Type)) + + " " + strings.ToUpper(rr.FingerPrint) +} + +type KEY struct { + DNSKEY +} + +type CDNSKEY struct { + DNSKEY +} + +type DNSKEY struct { + Hdr RR_Header + Flags uint16 + Protocol uint8 + Algorithm uint8 + PublicKey string `dns:"base64"` +} + +func (rr *DNSKEY) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Flags)) + + " " + strconv.Itoa(int(rr.Protocol)) + + " " + strconv.Itoa(int(rr.Algorithm)) + + " " + rr.PublicKey +} + +type RKEY struct { + Hdr RR_Header + Flags uint16 + Protocol uint8 + Algorithm uint8 + PublicKey string `dns:"base64"` +} + +func (rr *RKEY) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Flags)) + + " " + strconv.Itoa(int(rr.Protocol)) + + " " + strconv.Itoa(int(rr.Algorithm)) + + " " + rr.PublicKey +} + +type NSAPPTR struct { + Hdr RR_Header + Ptr string `dns:"domain-name"` +} + +func (rr *NSAPPTR) String() string { return rr.Hdr.String() + sprintName(rr.Ptr) } + +type NSEC3 struct { + Hdr RR_Header + Hash uint8 + Flags uint8 + Iterations uint16 + SaltLength uint8 + Salt string `dns:"size-hex:SaltLength"` + HashLength uint8 + NextDomain string `dns:"size-base32:HashLength"` + TypeBitMap []uint16 `dns:"nsec"` +} + +func (rr *NSEC3) String() string { + s := rr.Hdr.String() + s += strconv.Itoa(int(rr.Hash)) + + " " + strconv.Itoa(int(rr.Flags)) + + " " + strconv.Itoa(int(rr.Iterations)) + + " " + saltToString(rr.Salt) + + " " + rr.NextDomain + for i := 0; i < len(rr.TypeBitMap); i++ { + s += " " + Type(rr.TypeBitMap[i]).String() + } + return s +} + +func (rr *NSEC3) len() int { + l := rr.Hdr.len() + 6 + len(rr.Salt)/2 + 1 + len(rr.NextDomain) + 1 + lastwindow := uint32(2 ^ 32 + 1) + for _, t := range rr.TypeBitMap { + window := t / 256 + if uint32(window) != lastwindow { + l += 1 + 32 + } + lastwindow = uint32(window) + } + return l +} + +type NSEC3PARAM struct { + Hdr RR_Header + Hash uint8 + Flags uint8 + Iterations uint16 + SaltLength uint8 + Salt string `dns:"size-hex:SaltLength"` +} + +func (rr *NSEC3PARAM) String() string { + s := rr.Hdr.String() + s += strconv.Itoa(int(rr.Hash)) + + " " + strconv.Itoa(int(rr.Flags)) + + " " + strconv.Itoa(int(rr.Iterations)) + + " " + saltToString(rr.Salt) + return s +} + +type TKEY struct { + Hdr RR_Header + Algorithm string `dns:"domain-name"` + Inception uint32 + Expiration uint32 + Mode uint16 + Error uint16 + KeySize uint16 + Key string + OtherLen uint16 + OtherData string +} + +func (rr *TKEY) String() string { + // It has no presentation format + return "" +} + +// RFC3597 represents an unknown/generic RR. +type RFC3597 struct { + Hdr RR_Header + Rdata string `dns:"hex"` +} + +func (rr *RFC3597) String() string { + // Let's call it a hack + s := rfc3597Header(rr.Hdr) + + s += "\\# " + strconv.Itoa(len(rr.Rdata)/2) + " " + rr.Rdata + return s +} + +func rfc3597Header(h RR_Header) string { + var s string + + s += sprintName(h.Name) + "\t" + s += strconv.FormatInt(int64(h.Ttl), 10) + "\t" + s += "CLASS" + strconv.Itoa(int(h.Class)) + "\t" + s += "TYPE" + strconv.Itoa(int(h.Rrtype)) + "\t" + return s +} + +type URI struct { + Hdr RR_Header + Priority uint16 + Weight uint16 + Target string `dns:"octet"` +} + +func (rr *URI) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Priority)) + + " " + strconv.Itoa(int(rr.Weight)) + " " + sprintTxtOctet(rr.Target) +} + +type DHCID struct { + Hdr RR_Header + Digest string `dns:"base64"` +} + +func (rr *DHCID) String() string { return rr.Hdr.String() + rr.Digest } + +type TLSA struct { + Hdr RR_Header + Usage uint8 + Selector uint8 + MatchingType uint8 + Certificate string `dns:"hex"` +} + +func (rr *TLSA) String() string { + return rr.Hdr.String() + + strconv.Itoa(int(rr.Usage)) + + " " + strconv.Itoa(int(rr.Selector)) + + " " + strconv.Itoa(int(rr.MatchingType)) + + " " + rr.Certificate +} + +type SMIMEA struct { + Hdr RR_Header + Usage uint8 + Selector uint8 + MatchingType uint8 + Certificate string `dns:"hex"` +} + +func (rr *SMIMEA) String() string { + s := rr.Hdr.String() + + strconv.Itoa(int(rr.Usage)) + + " " + strconv.Itoa(int(rr.Selector)) + + " " + strconv.Itoa(int(rr.MatchingType)) + + // Every Nth char needs a space on this output. If we output + // this as one giant line, we can't read it can in because in some cases + // the cert length overflows scan.maxTok (2048). + sx := splitN(rr.Certificate, 1024) // conservative value here + s += " " + strings.Join(sx, " ") + return s +} + +type HIP struct { + Hdr RR_Header + HitLength uint8 + PublicKeyAlgorithm uint8 + PublicKeyLength uint16 + Hit string `dns:"size-hex:HitLength"` + PublicKey string `dns:"size-base64:PublicKeyLength"` + RendezvousServers []string `dns:"domain-name"` +} + +func (rr *HIP) String() string { + s := rr.Hdr.String() + + strconv.Itoa(int(rr.PublicKeyAlgorithm)) + + " " + rr.Hit + + " " + rr.PublicKey + for _, d := range rr.RendezvousServers { + s += " " + sprintName(d) + } + return s +} + +type NINFO struct { + Hdr RR_Header + ZSData []string `dns:"txt"` +} + +func (rr *NINFO) String() string { return rr.Hdr.String() + sprintTxt(rr.ZSData) } + +type NID struct { + Hdr RR_Header + Preference uint16 + NodeID uint64 +} + +func (rr *NID) String() string { + s := rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + node := fmt.Sprintf("%0.16x", rr.NodeID) + s += " " + node[0:4] + ":" + node[4:8] + ":" + node[8:12] + ":" + node[12:16] + return s +} + +type L32 struct { + Hdr RR_Header + Preference uint16 + Locator32 net.IP `dns:"a"` +} + +func (rr *L32) String() string { + if rr.Locator32 == nil { + return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + } + return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + + " " + rr.Locator32.String() +} + +type L64 struct { + Hdr RR_Header + Preference uint16 + Locator64 uint64 +} + +func (rr *L64) String() string { + s := rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + node := fmt.Sprintf("%0.16X", rr.Locator64) + s += " " + node[0:4] + ":" + node[4:8] + ":" + node[8:12] + ":" + node[12:16] + return s +} + +type LP struct { + Hdr RR_Header + Preference uint16 + Fqdn string `dns:"domain-name"` +} + +func (rr *LP) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Fqdn) +} + +type EUI48 struct { + Hdr RR_Header + Address uint64 `dns:"uint48"` +} + +func (rr *EUI48) String() string { return rr.Hdr.String() + euiToString(rr.Address, 48) } + +type EUI64 struct { + Hdr RR_Header + Address uint64 +} + +func (rr *EUI64) String() string { return rr.Hdr.String() + euiToString(rr.Address, 64) } + +type CAA struct { + Hdr RR_Header + Flag uint8 + Tag string + Value string `dns:"octet"` +} + +func (rr *CAA) String() string { + return rr.Hdr.String() + strconv.Itoa(int(rr.Flag)) + " " + rr.Tag + " " + sprintTxtOctet(rr.Value) +} + +type UID struct { + Hdr RR_Header + Uid uint32 +} + +func (rr *UID) String() string { return rr.Hdr.String() + strconv.FormatInt(int64(rr.Uid), 10) } + +type GID struct { + Hdr RR_Header + Gid uint32 +} + +func (rr *GID) String() string { return rr.Hdr.String() + strconv.FormatInt(int64(rr.Gid), 10) } + +type UINFO struct { + Hdr RR_Header + Uinfo string +} + +func (rr *UINFO) String() string { return rr.Hdr.String() + sprintTxt([]string{rr.Uinfo}) } + +type EID struct { + Hdr RR_Header + Endpoint string `dns:"hex"` +} + +func (rr *EID) String() string { return rr.Hdr.String() + strings.ToUpper(rr.Endpoint) } + +type NIMLOC struct { + Hdr RR_Header + Locator string `dns:"hex"` +} + +func (rr *NIMLOC) String() string { return rr.Hdr.String() + strings.ToUpper(rr.Locator) } + +type OPENPGPKEY struct { + Hdr RR_Header + PublicKey string `dns:"base64"` +} + +func (rr *OPENPGPKEY) String() string { return rr.Hdr.String() + rr.PublicKey } + +// TimeToString translates the RRSIG's incep. and expir. times to the +// string representation used when printing the record. +// It takes serial arithmetic (RFC 1982) into account. +func TimeToString(t uint32) string { + mod := ((int64(t) - time.Now().Unix()) / year68) - 1 + if mod < 0 { + mod = 0 + } + ti := time.Unix(int64(t)-(mod*year68), 0).UTC() + return ti.Format("20060102150405") +} + +// StringToTime translates the RRSIG's incep. and expir. times from +// string values like "20110403154150" to an 32 bit integer. +// It takes serial arithmetic (RFC 1982) into account. +func StringToTime(s string) (uint32, error) { + t, err := time.Parse("20060102150405", s) + if err != nil { + return 0, err + } + mod := (t.Unix() / year68) - 1 + if mod < 0 { + mod = 0 + } + return uint32(t.Unix() - (mod * year68)), nil +} + +// saltToString converts a NSECX salt to uppercase and returns "-" when it is empty. +func saltToString(s string) string { + if len(s) == 0 { + return "-" + } + return strings.ToUpper(s) +} + +func euiToString(eui uint64, bits int) (hex string) { + switch bits { + case 64: + hex = fmt.Sprintf("%16.16x", eui) + hex = hex[0:2] + "-" + hex[2:4] + "-" + hex[4:6] + "-" + hex[6:8] + + "-" + hex[8:10] + "-" + hex[10:12] + "-" + hex[12:14] + "-" + hex[14:16] + case 48: + hex = fmt.Sprintf("%12.12x", eui) + hex = hex[0:2] + "-" + hex[2:4] + "-" + hex[4:6] + "-" + hex[6:8] + + "-" + hex[8:10] + "-" + hex[10:12] + } + return +} + +// copyIP returns a copy of ip. +func copyIP(ip net.IP) net.IP { + p := make(net.IP, len(ip)) + copy(p, ip) + return p +} + +// SplitN splits a string into N sized string chunks. +// This might become an exported function once. +func splitN(s string, n int) []string { + if len(s) < n { + return []string{s} + } + sx := []string{} + p, i := 0, n + for { + if i <= len(s) { + sx = append(sx, s[p:i]) + } else { + sx = append(sx, s[p:]) + break + + } + p, i = p+n, i+n + } + + return sx +} diff --git a/vendor/github.com/miekg/dns/types_generate.go b/vendor/github.com/miekg/dns/types_generate.go new file mode 100644 index 00000000..dd131094 --- /dev/null +++ b/vendor/github.com/miekg/dns/types_generate.go @@ -0,0 +1,271 @@ +//+build ignore + +// types_generate.go is meant to run with go generate. It will use +// go/{importer,types} to track down all the RR struct types. Then for each type +// it will generate conversion tables (TypeToRR and TypeToString) and banal +// methods (len, Header, copy) based on the struct tags. The generated source is +// written to ztypes.go, and is meant to be checked into git. +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/importer" + "go/types" + "log" + "os" + "strings" + "text/template" +) + +var skipLen = map[string]struct{}{ + "NSEC": {}, + "NSEC3": {}, + "OPT": {}, +} + +var packageHdr = ` +// *** DO NOT MODIFY *** +// AUTOGENERATED BY go generate from type_generate.go + +package dns + +import ( + "encoding/base64" + "net" +) + +` + +var TypeToRR = template.Must(template.New("TypeToRR").Parse(` +// TypeToRR is a map of constructors for each RR type. +var TypeToRR = map[uint16]func() RR{ +{{range .}}{{if ne . "RFC3597"}} Type{{.}}: func() RR { return new({{.}}) }, +{{end}}{{end}} } + +`)) + +var typeToString = template.Must(template.New("typeToString").Parse(` +// TypeToString is a map of strings for each RR type. +var TypeToString = map[uint16]string{ +{{range .}}{{if ne . "NSAPPTR"}} Type{{.}}: "{{.}}", +{{end}}{{end}} TypeNSAPPTR: "NSAP-PTR", +} + +`)) + +var headerFunc = template.Must(template.New("headerFunc").Parse(` +// Header() functions +{{range .}} func (rr *{{.}}) Header() *RR_Header { return &rr.Hdr } +{{end}} + +`)) + +// getTypeStruct will take a type and the package scope, and return the +// (innermost) struct if the type is considered a RR type (currently defined as +// those structs beginning with a RR_Header, could be redefined as implementing +// the RR interface). The bool return value indicates if embedded structs were +// resolved. +func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { + st, ok := t.Underlying().(*types.Struct) + if !ok { + return nil, false + } + if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { + return st, false + } + if st.Field(0).Anonymous() { + st, _ := getTypeStruct(st.Field(0).Type(), scope) + return st, true + } + return nil, false +} + +func main() { + // Import and type-check the package + pkg, err := importer.Default().Import("github.com/miekg/dns") + fatalIfErr(err) + scope := pkg.Scope() + + // Collect constants like TypeX + var numberedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + b, ok := o.Type().(*types.Basic) + if !ok || b.Kind() != types.Uint16 { + continue + } + if !strings.HasPrefix(o.Name(), "Type") { + continue + } + name := strings.TrimPrefix(o.Name(), "Type") + if name == "PrivateRR" { + continue + } + numberedTypes = append(numberedTypes, name) + } + + // Collect actual types (*X) + var namedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + if st, _ := getTypeStruct(o.Type(), scope); st == nil { + continue + } + if name == "PrivateRR" { + continue + } + + // Check if corresponding TypeX exists + if scope.Lookup("Type"+o.Name()) == nil && o.Name() != "RFC3597" { + log.Fatalf("Constant Type%s does not exist.", o.Name()) + } + + namedTypes = append(namedTypes, o.Name()) + } + + b := &bytes.Buffer{} + b.WriteString(packageHdr) + + // Generate TypeToRR + fatalIfErr(TypeToRR.Execute(b, namedTypes)) + + // Generate typeToString + fatalIfErr(typeToString.Execute(b, numberedTypes)) + + // Generate headerFunc + fatalIfErr(headerFunc.Execute(b, namedTypes)) + + // Generate len() + fmt.Fprint(b, "// len() functions\n") + for _, name := range namedTypes { + if _, ok := skipLen[name]; ok { + continue + } + o := scope.Lookup(name) + st, isEmbedded := getTypeStruct(o.Type(), scope) + if isEmbedded { + continue + } + fmt.Fprintf(b, "func (rr *%s) len() int {\n", name) + fmt.Fprintf(b, "l := rr.Hdr.len()\n") + for i := 1; i < st.NumFields(); i++ { + o := func(s string) { fmt.Fprintf(b, s, st.Field(i).Name()) } + + if _, ok := st.Field(i).Type().(*types.Slice); ok { + switch st.Tag(i) { + case `dns:"-"`: + // ignored + case `dns:"cdomain-name"`, `dns:"domain-name"`, `dns:"txt"`: + o("for _, x := range rr.%s { l += len(x) + 1 }\n") + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + continue + } + + switch { + case st.Tag(i) == `dns:"-"`: + // ignored + case st.Tag(i) == `dns:"cdomain-name"`, st.Tag(i) == `dns:"domain-name"`: + o("l += len(rr.%s) + 1\n") + case st.Tag(i) == `dns:"octet"`: + o("l += len(rr.%s)\n") + case strings.HasPrefix(st.Tag(i), `dns:"size-base64`): + fallthrough + case st.Tag(i) == `dns:"base64"`: + o("l += base64.StdEncoding.DecodedLen(len(rr.%s))\n") + case strings.HasPrefix(st.Tag(i), `dns:"size-hex`): + fallthrough + case st.Tag(i) == `dns:"hex"`: + o("l += len(rr.%s)/2 + 1\n") + case st.Tag(i) == `dns:"a"`: + o("l += net.IPv4len // %s\n") + case st.Tag(i) == `dns:"aaaa"`: + o("l += net.IPv6len // %s\n") + case st.Tag(i) == `dns:"txt"`: + o("for _, t := range rr.%s { l += len(t) + 1 }\n") + case st.Tag(i) == `dns:"uint48"`: + o("l += 6 // %s\n") + case st.Tag(i) == "": + switch st.Field(i).Type().(*types.Basic).Kind() { + case types.Uint8: + o("l++ // %s\n") + case types.Uint16: + o("l += 2 // %s\n") + case types.Uint32: + o("l += 4 // %s\n") + case types.Uint64: + o("l += 8 // %s\n") + case types.String: + o("l += len(rr.%s) + 1\n") + default: + log.Fatalln(name, st.Field(i).Name()) + } + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + } + fmt.Fprintf(b, "return l }\n") + } + + // Generate copy() + fmt.Fprint(b, "// copy() functions\n") + for _, name := range namedTypes { + o := scope.Lookup(name) + st, isEmbedded := getTypeStruct(o.Type(), scope) + if isEmbedded { + continue + } + fmt.Fprintf(b, "func (rr *%s) copy() RR {\n", name) + fields := []string{"*rr.Hdr.copyHeader()"} + for i := 1; i < st.NumFields(); i++ { + f := st.Field(i).Name() + if sl, ok := st.Field(i).Type().(*types.Slice); ok { + t := sl.Underlying().String() + t = strings.TrimPrefix(t, "[]") + if strings.Contains(t, ".") { + splits := strings.Split(t, ".") + t = splits[len(splits)-1] + } + fmt.Fprintf(b, "%s := make([]%s, len(rr.%s)); copy(%s, rr.%s)\n", + f, t, f, f, f) + fields = append(fields, f) + continue + } + if st.Field(i).Type().String() == "net.IP" { + fields = append(fields, "copyIP(rr."+f+")") + continue + } + fields = append(fields, "rr."+f) + } + fmt.Fprintf(b, "return &%s{%s}\n", name, strings.Join(fields, ",")) + fmt.Fprintf(b, "}\n") + } + + // gofmt + res, err := format.Source(b.Bytes()) + if err != nil { + b.WriteTo(os.Stderr) + log.Fatal(err) + } + + // write result + f, err := os.Create("ztypes.go") + fatalIfErr(err) + defer f.Close() + f.Write(res) +} + +func fatalIfErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/vendor/github.com/miekg/dns/udp.go b/vendor/github.com/miekg/dns/udp.go new file mode 100644 index 00000000..af111b9a --- /dev/null +++ b/vendor/github.com/miekg/dns/udp.go @@ -0,0 +1,34 @@ +// +build !windows + +package dns + +import ( + "net" +) + +// SessionUDP holds the remote address and the associated +// out-of-band data. +type SessionUDP struct { + raddr *net.UDPAddr + context []byte +} + +// RemoteAddr returns the remote network address. +func (s *SessionUDP) RemoteAddr() net.Addr { return s.raddr } + +// ReadFromSessionUDP acts just like net.UDPConn.ReadFrom(), but returns a session object instead of a +// net.UDPAddr. +func ReadFromSessionUDP(conn *net.UDPConn, b []byte) (int, *SessionUDP, error) { + oob := make([]byte, 40) + n, oobn, _, raddr, err := conn.ReadMsgUDP(b, oob) + if err != nil { + return n, nil, err + } + return n, &SessionUDP{raddr, oob[:oobn]}, err +} + +// WriteToSessionUDP acts just like net.UDPConn.WritetTo(), but uses a *SessionUDP instead of a net.Addr. +func WriteToSessionUDP(conn *net.UDPConn, b []byte, session *SessionUDP) (int, error) { + n, _, err := conn.WriteMsgUDP(b, session.context, session.raddr) + return n, err +} diff --git a/vendor/github.com/miekg/dns/udp_linux.go b/vendor/github.com/miekg/dns/udp_linux.go new file mode 100644 index 00000000..033df423 --- /dev/null +++ b/vendor/github.com/miekg/dns/udp_linux.go @@ -0,0 +1,105 @@ +// +build linux,!appengine + +package dns + +// See: +// * http://stackoverflow.com/questions/3062205/setting-the-source-ip-for-a-udp-socket and +// * http://blog.powerdns.com/2012/10/08/on-binding-datagram-udp-sockets-to-the-any-addresses/ +// +// Why do we need this: When listening on 0.0.0.0 with UDP so kernel decides what is the outgoing +// interface, this might not always be the correct one. This code will make sure the egress +// packet's interface matched the ingress' one. + +import ( + "net" + "syscall" +) + +// setUDPSocketOptions sets the UDP socket options. +// This function is implemented on a per platform basis. See udp_*.go for more details +func setUDPSocketOptions(conn *net.UDPConn) error { + sa, err := getUDPSocketName(conn) + if err != nil { + return err + } + switch sa.(type) { + case *syscall.SockaddrInet6: + v6only, err := getUDPSocketOptions6Only(conn) + if err != nil { + return err + } + setUDPSocketOptions6(conn) + if !v6only { + setUDPSocketOptions4(conn) + } + case *syscall.SockaddrInet4: + setUDPSocketOptions4(conn) + } + return nil +} + +// setUDPSocketOptions4 prepares the v4 socket for sessions. +func setUDPSocketOptions4(conn *net.UDPConn) error { + file, err := conn.File() + if err != nil { + return err + } + if err := syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_IP, syscall.IP_PKTINFO, 1); err != nil { + file.Close() + return err + } + // Calling File() above results in the connection becoming blocking, we must fix that. + // See https://github.com/miekg/dns/issues/279 + err = syscall.SetNonblock(int(file.Fd()), true) + if err != nil { + file.Close() + return err + } + file.Close() + return nil +} + +// setUDPSocketOptions6 prepares the v6 socket for sessions. +func setUDPSocketOptions6(conn *net.UDPConn) error { + file, err := conn.File() + if err != nil { + return err + } + if err := syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_IPV6, syscall.IPV6_RECVPKTINFO, 1); err != nil { + file.Close() + return err + } + err = syscall.SetNonblock(int(file.Fd()), true) + if err != nil { + file.Close() + return err + } + file.Close() + return nil +} + +// getUDPSocketOption6Only return true if the socket is v6 only and false when it is v4/v6 combined +// (dualstack). +func getUDPSocketOptions6Only(conn *net.UDPConn) (bool, error) { + file, err := conn.File() + if err != nil { + return false, err + } + // dual stack. See http://stackoverflow.com/questions/1618240/how-to-support-both-ipv4-and-ipv6-connections + v6only, err := syscall.GetsockoptInt(int(file.Fd()), syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY) + if err != nil { + file.Close() + return false, err + } + file.Close() + return v6only == 1, nil +} + +func getUDPSocketName(conn *net.UDPConn) (syscall.Sockaddr, error) { + file, err := conn.File() + if err != nil { + return nil, err + } + defer file.Close() + return syscall.Getsockname(int(file.Fd())) +} diff --git a/vendor/github.com/miekg/dns/udp_other.go b/vendor/github.com/miekg/dns/udp_other.go new file mode 100644 index 00000000..488a282b --- /dev/null +++ b/vendor/github.com/miekg/dns/udp_other.go @@ -0,0 +1,15 @@ +// +build !linux appengine + +package dns + +import ( + "net" +) + +// These do nothing. See udp_linux.go for an example of how to implement this. + +// We tried to adhire to some kind of naming scheme. +func setUDPSocketOptions(conn *net.UDPConn) error { return nil } +func setUDPSocketOptions4(conn *net.UDPConn) error { return nil } +func setUDPSocketOptions6(conn *net.UDPConn) error { return nil } +func getUDPSocketOptions6Only(conn *net.UDPConn) (bool, error) { return false, nil } diff --git a/vendor/github.com/miekg/dns/udp_windows.go b/vendor/github.com/miekg/dns/udp_windows.go new file mode 100644 index 00000000..51e532ac --- /dev/null +++ b/vendor/github.com/miekg/dns/udp_windows.go @@ -0,0 +1,29 @@ +// +build windows + +package dns + +import "net" + +type SessionUDP struct { + raddr *net.UDPAddr +} + +func (s *SessionUDP) RemoteAddr() net.Addr { return s.raddr } + +// ReadFromSessionUDP acts just like net.UDPConn.ReadFrom(), but returns a session object instead of a +// net.UDPAddr. +func ReadFromSessionUDP(conn *net.UDPConn, b []byte) (int, *SessionUDP, error) { + n, raddr, err := conn.ReadFrom(b) + if err != nil { + return n, nil, err + } + session := &SessionUDP{raddr.(*net.UDPAddr)} + return n, session, err +} + +// WriteToSessionUDP acts just like net.UDPConn.WritetTo(), but uses a *SessionUDP instead of a net.Addr. +func WriteToSessionUDP(conn *net.UDPConn, b []byte, session *SessionUDP) (int, error) { + n, err := conn.WriteTo(b, session.raddr) + return n, err +} + diff --git a/vendor/github.com/miekg/dns/update.go b/vendor/github.com/miekg/dns/update.go new file mode 100644 index 00000000..e90c5c96 --- /dev/null +++ b/vendor/github.com/miekg/dns/update.go @@ -0,0 +1,106 @@ +package dns + +// NameUsed sets the RRs in the prereq section to +// "Name is in use" RRs. RFC 2136 section 2.4.4. +func (u *Msg) NameUsed(rr []RR) { + if u.Answer == nil { + u.Answer = make([]RR, 0, len(rr)) + } + for _, r := range rr { + u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: TypeANY, Class: ClassANY}}) + } +} + +// NameNotUsed sets the RRs in the prereq section to +// "Name is in not use" RRs. RFC 2136 section 2.4.5. +func (u *Msg) NameNotUsed(rr []RR) { + if u.Answer == nil { + u.Answer = make([]RR, 0, len(rr)) + } + for _, r := range rr { + u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: TypeANY, Class: ClassNONE}}) + } +} + +// Used sets the RRs in the prereq section to +// "RRset exists (value dependent -- with rdata)" RRs. RFC 2136 section 2.4.2. +func (u *Msg) Used(rr []RR) { + if len(u.Question) == 0 { + panic("dns: empty question section") + } + if u.Answer == nil { + u.Answer = make([]RR, 0, len(rr)) + } + for _, r := range rr { + r.Header().Class = u.Question[0].Qclass + u.Answer = append(u.Answer, r) + } +} + +// RRsetUsed sets the RRs in the prereq section to +// "RRset exists (value independent -- no rdata)" RRs. RFC 2136 section 2.4.1. +func (u *Msg) RRsetUsed(rr []RR) { + if u.Answer == nil { + u.Answer = make([]RR, 0, len(rr)) + } + for _, r := range rr { + u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: r.Header().Rrtype, Class: ClassANY}}) + } +} + +// RRsetNotUsed sets the RRs in the prereq section to +// "RRset does not exist" RRs. RFC 2136 section 2.4.3. +func (u *Msg) RRsetNotUsed(rr []RR) { + if u.Answer == nil { + u.Answer = make([]RR, 0, len(rr)) + } + for _, r := range rr { + u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: r.Header().Rrtype, Class: ClassNONE}}) + } +} + +// Insert creates a dynamic update packet that adds an complete RRset, see RFC 2136 section 2.5.1. +func (u *Msg) Insert(rr []RR) { + if len(u.Question) == 0 { + panic("dns: empty question section") + } + if u.Ns == nil { + u.Ns = make([]RR, 0, len(rr)) + } + for _, r := range rr { + r.Header().Class = u.Question[0].Qclass + u.Ns = append(u.Ns, r) + } +} + +// RemoveRRset creates a dynamic update packet that deletes an RRset, see RFC 2136 section 2.5.2. +func (u *Msg) RemoveRRset(rr []RR) { + if u.Ns == nil { + u.Ns = make([]RR, 0, len(rr)) + } + for _, r := range rr { + u.Ns = append(u.Ns, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: r.Header().Rrtype, Class: ClassANY}}) + } +} + +// RemoveName creates a dynamic update packet that deletes all RRsets of a name, see RFC 2136 section 2.5.3 +func (u *Msg) RemoveName(rr []RR) { + if u.Ns == nil { + u.Ns = make([]RR, 0, len(rr)) + } + for _, r := range rr { + u.Ns = append(u.Ns, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: TypeANY, Class: ClassANY}}) + } +} + +// Remove creates a dynamic update packet deletes RR from a RRSset, see RFC 2136 section 2.5.4 +func (u *Msg) Remove(rr []RR) { + if u.Ns == nil { + u.Ns = make([]RR, 0, len(rr)) + } + for _, r := range rr { + r.Header().Class = ClassNONE + r.Header().Ttl = 0 + u.Ns = append(u.Ns, r) + } +} diff --git a/vendor/github.com/miekg/dns/xfr.go b/vendor/github.com/miekg/dns/xfr.go new file mode 100644 index 00000000..7346deff --- /dev/null +++ b/vendor/github.com/miekg/dns/xfr.go @@ -0,0 +1,244 @@ +package dns + +import ( + "time" +) + +// Envelope is used when doing a zone transfer with a remote server. +type Envelope struct { + RR []RR // The set of RRs in the answer section of the xfr reply message. + Error error // If something went wrong, this contains the error. +} + +// A Transfer defines parameters that are used during a zone transfer. +type Transfer struct { + *Conn + DialTimeout time.Duration // net.DialTimeout, defaults to 2 seconds + ReadTimeout time.Duration // net.Conn.SetReadTimeout value for connections, defaults to 2 seconds + WriteTimeout time.Duration // net.Conn.SetWriteTimeout value for connections, defaults to 2 seconds + TsigSecret map[string]string // Secret(s) for Tsig map[], zonename must be fully qualified + tsigTimersOnly bool +} + +// Think we need to away to stop the transfer + +// In performs an incoming transfer with the server in a. +// If you would like to set the source IP, or some other attribute +// of a Dialer for a Transfer, you can do so by specifying the attributes +// in the Transfer.Conn: +// +// d := net.Dialer{LocalAddr: transfer_source} +// con, err := d.Dial("tcp", master) +// dnscon := &dns.Conn{Conn:con} +// transfer = &dns.Transfer{Conn: dnscon} +// channel, err := transfer.In(message, master) +// +func (t *Transfer) In(q *Msg, a string) (env chan *Envelope, err error) { + timeout := dnsTimeout + if t.DialTimeout != 0 { + timeout = t.DialTimeout + } + if t.Conn == nil { + t.Conn, err = DialTimeout("tcp", a, timeout) + if err != nil { + return nil, err + } + } + if err := t.WriteMsg(q); err != nil { + return nil, err + } + env = make(chan *Envelope) + go func() { + if q.Question[0].Qtype == TypeAXFR { + go t.inAxfr(q.Id, env) + return + } + if q.Question[0].Qtype == TypeIXFR { + go t.inIxfr(q.Id, env) + return + } + }() + return env, nil +} + +func (t *Transfer) inAxfr(id uint16, c chan *Envelope) { + first := true + defer t.Close() + defer close(c) + timeout := dnsTimeout + if t.ReadTimeout != 0 { + timeout = t.ReadTimeout + } + for { + t.Conn.SetReadDeadline(time.Now().Add(timeout)) + in, err := t.ReadMsg() + if err != nil { + c <- &Envelope{nil, err} + return + } + if id != in.Id { + c <- &Envelope{in.Answer, ErrId} + return + } + if first { + if !isSOAFirst(in) { + c <- &Envelope{in.Answer, ErrSoa} + return + } + first = !first + // only one answer that is SOA, receive more + if len(in.Answer) == 1 { + t.tsigTimersOnly = true + c <- &Envelope{in.Answer, nil} + continue + } + } + + if !first { + t.tsigTimersOnly = true // Subsequent envelopes use this. + if isSOALast(in) { + c <- &Envelope{in.Answer, nil} + return + } + c <- &Envelope{in.Answer, nil} + } + } +} + +func (t *Transfer) inIxfr(id uint16, c chan *Envelope) { + serial := uint32(0) // The first serial seen is the current server serial + first := true + defer t.Close() + defer close(c) + timeout := dnsTimeout + if t.ReadTimeout != 0 { + timeout = t.ReadTimeout + } + for { + t.SetReadDeadline(time.Now().Add(timeout)) + in, err := t.ReadMsg() + if err != nil { + c <- &Envelope{nil, err} + return + } + if id != in.Id { + c <- &Envelope{in.Answer, ErrId} + return + } + if first { + // A single SOA RR signals "no changes" + if len(in.Answer) == 1 && isSOAFirst(in) { + c <- &Envelope{in.Answer, nil} + return + } + + // Check if the returned answer is ok + if !isSOAFirst(in) { + c <- &Envelope{in.Answer, ErrSoa} + return + } + // This serial is important + serial = in.Answer[0].(*SOA).Serial + first = !first + } + + // Now we need to check each message for SOA records, to see what we need to do + if !first { + t.tsigTimersOnly = true + // If the last record in the IXFR contains the servers' SOA, we should quit + if v, ok := in.Answer[len(in.Answer)-1].(*SOA); ok { + if v.Serial == serial { + c <- &Envelope{in.Answer, nil} + return + } + } + c <- &Envelope{in.Answer, nil} + } + } +} + +// Out performs an outgoing transfer with the client connecting in w. +// Basic use pattern: +// +// ch := make(chan *dns.Envelope) +// tr := new(dns.Transfer) +// go tr.Out(w, r, ch) +// ch <- &dns.Envelope{RR: []dns.RR{soa, rr1, rr2, rr3, soa}} +// close(ch) +// w.Hijack() +// // w.Close() // Client closes connection +// +// The server is responsible for sending the correct sequence of RRs through the +// channel ch. +func (t *Transfer) Out(w ResponseWriter, q *Msg, ch chan *Envelope) error { + for x := range ch { + r := new(Msg) + // Compress? + r.SetReply(q) + r.Authoritative = true + // assume it fits TODO(miek): fix + r.Answer = append(r.Answer, x.RR...) + if err := w.WriteMsg(r); err != nil { + return err + } + } + w.TsigTimersOnly(true) + return nil +} + +// ReadMsg reads a message from the transfer connection t. +func (t *Transfer) ReadMsg() (*Msg, error) { + m := new(Msg) + p := make([]byte, MaxMsgSize) + n, err := t.Read(p) + if err != nil && n == 0 { + return nil, err + } + p = p[:n] + if err := m.Unpack(p); err != nil { + return nil, err + } + if ts := m.IsTsig(); ts != nil && t.TsigSecret != nil { + if _, ok := t.TsigSecret[ts.Hdr.Name]; !ok { + return m, ErrSecret + } + // Need to work on the original message p, as that was used to calculate the tsig. + err = TsigVerify(p, t.TsigSecret[ts.Hdr.Name], t.tsigRequestMAC, t.tsigTimersOnly) + t.tsigRequestMAC = ts.MAC + } + return m, err +} + +// WriteMsg writes a message through the transfer connection t. +func (t *Transfer) WriteMsg(m *Msg) (err error) { + var out []byte + if ts := m.IsTsig(); ts != nil && t.TsigSecret != nil { + if _, ok := t.TsigSecret[ts.Hdr.Name]; !ok { + return ErrSecret + } + out, t.tsigRequestMAC, err = TsigGenerate(m, t.TsigSecret[ts.Hdr.Name], t.tsigRequestMAC, t.tsigTimersOnly) + } else { + out, err = m.Pack() + } + if err != nil { + return err + } + if _, err = t.Write(out); err != nil { + return err + } + return nil +} + +func isSOAFirst(in *Msg) bool { + if len(in.Answer) > 0 { + return in.Answer[0].Header().Rrtype == TypeSOA + } + return false +} + +func isSOALast(in *Msg) bool { + if len(in.Answer) > 0 { + return in.Answer[len(in.Answer)-1].Header().Rrtype == TypeSOA + } + return false +} diff --git a/vendor/github.com/miekg/dns/zcompress.go b/vendor/github.com/miekg/dns/zcompress.go new file mode 100644 index 00000000..b277978b --- /dev/null +++ b/vendor/github.com/miekg/dns/zcompress.go @@ -0,0 +1,119 @@ +// *** DO NOT MODIFY *** +// AUTOGENERATED BY go generate from compress_generate.go + +package dns + +func compressionLenHelperType(c map[string]int, r RR) { + switch x := r.(type) { + case *PTR: + compressionLenHelper(c, x.Ptr) + case *SOA: + compressionLenHelper(c, x.Ns) + compressionLenHelper(c, x.Mbox) + case *AFSDB: + compressionLenHelper(c, x.Hostname) + case *HIP: + for i := range x.RendezvousServers { + compressionLenHelper(c, x.RendezvousServers[i]) + } + case *LP: + compressionLenHelper(c, x.Fqdn) + case *CNAME: + compressionLenHelper(c, x.Target) + case *MB: + compressionLenHelper(c, x.Mb) + case *RP: + compressionLenHelper(c, x.Mbox) + compressionLenHelper(c, x.Txt) + case *RRSIG: + compressionLenHelper(c, x.SignerName) + case *MF: + compressionLenHelper(c, x.Mf) + case *MINFO: + compressionLenHelper(c, x.Rmail) + compressionLenHelper(c, x.Email) + case *SIG: + compressionLenHelper(c, x.SignerName) + case *SRV: + compressionLenHelper(c, x.Target) + case *TSIG: + compressionLenHelper(c, x.Algorithm) + case *KX: + compressionLenHelper(c, x.Exchanger) + case *MG: + compressionLenHelper(c, x.Mg) + case *NSAPPTR: + compressionLenHelper(c, x.Ptr) + case *PX: + compressionLenHelper(c, x.Map822) + compressionLenHelper(c, x.Mapx400) + case *DNAME: + compressionLenHelper(c, x.Target) + case *MR: + compressionLenHelper(c, x.Mr) + case *MX: + compressionLenHelper(c, x.Mx) + case *TKEY: + compressionLenHelper(c, x.Algorithm) + case *NSEC: + compressionLenHelper(c, x.NextDomain) + case *TALINK: + compressionLenHelper(c, x.PreviousName) + compressionLenHelper(c, x.NextName) + case *MD: + compressionLenHelper(c, x.Md) + case *NAPTR: + compressionLenHelper(c, x.Replacement) + case *NS: + compressionLenHelper(c, x.Ns) + case *RT: + compressionLenHelper(c, x.Host) + } +} + +func compressionLenSearchType(c map[string]int, r RR) (int, bool) { + switch x := r.(type) { + case *MG: + k1, ok1 := compressionLenSearch(c, x.Mg) + return k1, ok1 + case *PTR: + k1, ok1 := compressionLenSearch(c, x.Ptr) + return k1, ok1 + case *AFSDB: + k1, ok1 := compressionLenSearch(c, x.Hostname) + return k1, ok1 + case *MB: + k1, ok1 := compressionLenSearch(c, x.Mb) + return k1, ok1 + case *MD: + k1, ok1 := compressionLenSearch(c, x.Md) + return k1, ok1 + case *MF: + k1, ok1 := compressionLenSearch(c, x.Mf) + return k1, ok1 + case *NS: + k1, ok1 := compressionLenSearch(c, x.Ns) + return k1, ok1 + case *RT: + k1, ok1 := compressionLenSearch(c, x.Host) + return k1, ok1 + case *SOA: + k1, ok1 := compressionLenSearch(c, x.Ns) + k2, ok2 := compressionLenSearch(c, x.Mbox) + return k1 + k2, ok1 && ok2 + case *CNAME: + k1, ok1 := compressionLenSearch(c, x.Target) + return k1, ok1 + case *MINFO: + k1, ok1 := compressionLenSearch(c, x.Rmail) + k2, ok2 := compressionLenSearch(c, x.Email) + return k1 + k2, ok1 && ok2 + case *MR: + k1, ok1 := compressionLenSearch(c, x.Mr) + return k1, ok1 + case *MX: + k1, ok1 := compressionLenSearch(c, x.Mx) + return k1, ok1 + } + return 0, false +} diff --git a/vendor/github.com/miekg/dns/zmsg.go b/vendor/github.com/miekg/dns/zmsg.go new file mode 100644 index 00000000..418fb1fe --- /dev/null +++ b/vendor/github.com/miekg/dns/zmsg.go @@ -0,0 +1,3565 @@ +// *** DO NOT MODIFY *** +// AUTOGENERATED BY go generate from msg_generate.go + +package dns + +// pack*() functions + +func (rr *A) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packDataA(rr.A, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *AAAA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packDataAAAA(rr.AAAA, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *AFSDB) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Subtype, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Hostname, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *ANY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *AVC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringTxt(rr.Txt, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *CAA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.Flag, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Tag, msg, off) + if err != nil { + return off, err + } + off, err = packStringOctet(rr.Value, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *CDNSKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Protocol, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.PublicKey, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *CDS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.DigestType, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.Digest, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *CERT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Type, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.Certificate, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *CNAME) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Target, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *DHCID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringBase64(rr.Digest, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *DLV) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.DigestType, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.Digest, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *DNAME) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Target, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *DNSKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Protocol, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.PublicKey, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *DS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.DigestType, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.Digest, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *EID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringHex(rr.Endpoint, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *EUI48) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint48(rr.Address, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *EUI64) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint64(rr.Address, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *GID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint32(rr.Gid, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *GPOS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packString(rr.Longitude, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Latitude, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Altitude, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *HINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packString(rr.Cpu, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Os, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *HIP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.HitLength, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.PublicKeyAlgorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.PublicKeyLength, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.Hit, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.PublicKey, msg, off) + if err != nil { + return off, err + } + off, err = packDataDomainNames(rr.RendezvousServers, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *KEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Protocol, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.PublicKey, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *KX) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Exchanger, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *L32) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = packDataA(rr.Locator32, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *L64) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = packUint64(rr.Locator64, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *LOC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.Version, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Size, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.HorizPre, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.VertPre, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Latitude, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Longitude, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Altitude, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *LP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Fqdn, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *MB) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Mb, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *MD) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Md, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *MF) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Mf, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *MG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Mg, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *MINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Rmail, msg, off, compression, compress) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Email, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *MR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Mr, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *MX) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Mx, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NAPTR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Order, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Service, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Regexp, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Replacement, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = packUint64(rr.NodeID, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NIMLOC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringHex(rr.Locator, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringTxt(rr.ZSData, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Ns, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NSAPPTR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Ptr, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NSEC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.NextDomain, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = packDataNsec(rr.TypeBitMap, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NSEC3) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.Hash, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Iterations, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.SaltLength, msg, off) + if err != nil { + return off, err + } + // Only pack salt if value is not "-", i.e. empty + if rr.Salt != "-" { + off, err = packStringHex(rr.Salt, msg, off) + if err != nil { + return off, err + } + } + off, err = packUint8(rr.HashLength, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase32(rr.NextDomain, msg, off) + if err != nil { + return off, err + } + off, err = packDataNsec(rr.TypeBitMap, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *NSEC3PARAM) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.Hash, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Iterations, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.SaltLength, msg, off) + if err != nil { + return off, err + } + // Only pack salt if value is not "-", i.e. empty + if rr.Salt != "-" { + off, err = packStringHex(rr.Salt, msg, off) + if err != nil { + return off, err + } + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *OPENPGPKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringBase64(rr.PublicKey, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *OPT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packDataOpt(rr.Option, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *PTR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Ptr, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *PX) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Map822, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Mapx400, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *RFC3597) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringHex(rr.Rdata, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *RKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Protocol, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.PublicKey, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *RP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Mbox, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Txt, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *RRSIG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.TypeCovered, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Labels, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.OrigTtl, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Expiration, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Inception, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.SignerName, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.Signature, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *RT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Preference, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Host, msg, off, compression, compress) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *SIG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.TypeCovered, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Labels, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.OrigTtl, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Expiration, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Inception, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.SignerName, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = packStringBase64(rr.Signature, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *SMIMEA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.Usage, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Selector, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.MatchingType, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.Certificate, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *SOA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Ns, msg, off, compression, compress) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Mbox, msg, off, compression, compress) + if err != nil { + return off, err + } + off, err = packUint32(rr.Serial, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Refresh, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Retry, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Expire, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Minttl, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *SPF) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringTxt(rr.Txt, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *SRV) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Priority, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Weight, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Port, msg, off) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.Target, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *SSHFP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Type, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.FingerPrint, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *TA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.KeyTag, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Algorithm, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.DigestType, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.Digest, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *TALINK) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.PreviousName, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = PackDomainName(rr.NextName, msg, off, compression, false) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *TKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Algorithm, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = packUint32(rr.Inception, msg, off) + if err != nil { + return off, err + } + off, err = packUint32(rr.Expiration, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Mode, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Error, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.KeySize, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.Key, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.OtherLen, msg, off) + if err != nil { + return off, err + } + off, err = packString(rr.OtherData, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *TLSA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint8(rr.Usage, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.Selector, msg, off) + if err != nil { + return off, err + } + off, err = packUint8(rr.MatchingType, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.Certificate, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *TSIG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = PackDomainName(rr.Algorithm, msg, off, compression, false) + if err != nil { + return off, err + } + off, err = packUint48(rr.TimeSigned, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Fudge, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.MACSize, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.MAC, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.OrigId, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Error, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.OtherLen, msg, off) + if err != nil { + return off, err + } + off, err = packStringHex(rr.OtherData, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *TXT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packStringTxt(rr.Txt, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *UID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint32(rr.Uid, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *UINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packString(rr.Uinfo, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *URI) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint16(rr.Priority, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Weight, msg, off) + if err != nil { + return off, err + } + off, err = packStringOctet(rr.Target, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +func (rr *X25) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packString(rr.PSDNAddress, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + +// unpack*() functions + +func unpackA(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(A) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.A, off, err = unpackDataA(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackAAAA(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(AAAA) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.AAAA, off, err = unpackDataAAAA(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackAFSDB(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(AFSDB) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Subtype, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Hostname, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackANY(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(ANY) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + return rr, off, err +} + +func unpackAVC(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(AVC) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Txt, off, err = unpackStringTxt(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackCAA(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(CAA) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Flag, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Tag, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Value, off, err = unpackStringOctet(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackCDNSKEY(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(CDNSKEY) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Flags, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Protocol, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackCDS(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(CDS) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.KeyTag, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.DigestType, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackCERT(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(CERT) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Type, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.KeyTag, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Certificate, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackCNAME(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(CNAME) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Target, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackDHCID(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(DHCID) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Digest, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackDLV(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(DLV) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.KeyTag, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.DigestType, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackDNAME(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(DNAME) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Target, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackDNSKEY(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(DNSKEY) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Flags, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Protocol, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackDS(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(DS) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.KeyTag, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.DigestType, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackEID(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(EID) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Endpoint, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackEUI48(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(EUI48) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Address, off, err = unpackUint48(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackEUI64(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(EUI64) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Address, off, err = unpackUint64(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackGID(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(GID) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Gid, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackGPOS(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(GPOS) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Longitude, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Latitude, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Altitude, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackHINFO(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(HINFO) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Cpu, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Os, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackHIP(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(HIP) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.HitLength, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.PublicKeyAlgorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.PublicKeyLength, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Hit, off, err = unpackStringHex(msg, off, off+int(rr.HitLength)) + if err != nil { + return rr, off, err + } + rr.PublicKey, off, err = unpackStringBase64(msg, off, off+int(rr.PublicKeyLength)) + if err != nil { + return rr, off, err + } + rr.RendezvousServers, off, err = unpackDataDomainNames(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackKEY(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(KEY) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Flags, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Protocol, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackKX(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(KX) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Exchanger, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackL32(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(L32) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Locator32, off, err = unpackDataA(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackL64(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(L64) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Locator64, off, err = unpackUint64(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackLOC(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(LOC) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Version, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Size, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.HorizPre, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.VertPre, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Latitude, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Longitude, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Altitude, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackLP(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(LP) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Fqdn, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackMB(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(MB) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Mb, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackMD(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(MD) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Md, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackMF(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(MF) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Mf, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackMG(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(MG) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Mg, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackMINFO(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(MINFO) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Rmail, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Email, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackMR(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(MR) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Mr, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackMX(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(MX) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Mx, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNAPTR(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NAPTR) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Order, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Flags, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Service, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Regexp, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Replacement, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNID(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NID) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.NodeID, off, err = unpackUint64(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNIMLOC(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NIMLOC) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Locator, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNINFO(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NINFO) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.ZSData, off, err = unpackStringTxt(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNS(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NS) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Ns, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNSAPPTR(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NSAPPTR) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Ptr, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNSEC(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NSEC) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.NextDomain, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.TypeBitMap, off, err = unpackDataNsec(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNSEC3(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NSEC3) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Hash, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Flags, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Iterations, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.SaltLength, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Salt, off, err = unpackStringHex(msg, off, off+int(rr.SaltLength)) + if err != nil { + return rr, off, err + } + rr.HashLength, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.NextDomain, off, err = unpackStringBase32(msg, off, off+int(rr.HashLength)) + if err != nil { + return rr, off, err + } + rr.TypeBitMap, off, err = unpackDataNsec(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackNSEC3PARAM(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(NSEC3PARAM) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Hash, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Flags, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Iterations, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.SaltLength, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Salt, off, err = unpackStringHex(msg, off, off+int(rr.SaltLength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackOPENPGPKEY(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(OPENPGPKEY) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackOPT(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(OPT) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Option, off, err = unpackDataOpt(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackPTR(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(PTR) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Ptr, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackPX(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(PX) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Map822, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Mapx400, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackRFC3597(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(RFC3597) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Rdata, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackRKEY(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(RKEY) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Flags, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Protocol, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackRP(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(RP) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Mbox, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Txt, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackRRSIG(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(RRSIG) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.TypeCovered, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Labels, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.OrigTtl, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Expiration, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Inception, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.KeyTag, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.SignerName, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Signature, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackRT(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(RT) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Preference, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Host, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackSIG(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(SIG) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.TypeCovered, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Labels, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.OrigTtl, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Expiration, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Inception, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.KeyTag, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.SignerName, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Signature, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackSMIMEA(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(SMIMEA) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Usage, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Selector, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.MatchingType, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Certificate, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackSOA(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(SOA) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Ns, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Mbox, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Serial, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Refresh, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Retry, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Expire, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Minttl, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackSPF(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(SPF) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Txt, off, err = unpackStringTxt(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackSRV(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(SRV) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Priority, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Weight, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Port, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Target, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackSSHFP(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(SSHFP) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Type, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.FingerPrint, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackTA(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(TA) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.KeyTag, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Algorithm, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.DigestType, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackTALINK(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(TALINK) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.PreviousName, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.NextName, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackTKEY(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(TKEY) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Algorithm, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Inception, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Expiration, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Mode, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Error, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.KeySize, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Key, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.OtherLen, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.OtherData, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackTLSA(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(TLSA) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Usage, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Selector, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.MatchingType, off, err = unpackUint8(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Certificate, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackTSIG(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(TSIG) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Algorithm, off, err = UnpackDomainName(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.TimeSigned, off, err = unpackUint48(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Fudge, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.MACSize, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.MAC, off, err = unpackStringHex(msg, off, off+int(rr.MACSize)) + if err != nil { + return rr, off, err + } + rr.OrigId, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Error, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.OtherLen, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.OtherData, off, err = unpackStringHex(msg, off, off+int(rr.OtherLen)) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackTXT(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(TXT) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Txt, off, err = unpackStringTxt(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackUID(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(UID) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Uid, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackUINFO(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(UINFO) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Uinfo, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackURI(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(URI) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Priority, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Weight, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Target, off, err = unpackStringOctet(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +func unpackX25(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(X25) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.PSDNAddress, off, err = unpackString(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + +var typeToUnpack = map[uint16]func(RR_Header, []byte, int) (RR, int, error){ + TypeA: unpackA, + TypeAAAA: unpackAAAA, + TypeAFSDB: unpackAFSDB, + TypeANY: unpackANY, + TypeAVC: unpackAVC, + TypeCAA: unpackCAA, + TypeCDNSKEY: unpackCDNSKEY, + TypeCDS: unpackCDS, + TypeCERT: unpackCERT, + TypeCNAME: unpackCNAME, + TypeDHCID: unpackDHCID, + TypeDLV: unpackDLV, + TypeDNAME: unpackDNAME, + TypeDNSKEY: unpackDNSKEY, + TypeDS: unpackDS, + TypeEID: unpackEID, + TypeEUI48: unpackEUI48, + TypeEUI64: unpackEUI64, + TypeGID: unpackGID, + TypeGPOS: unpackGPOS, + TypeHINFO: unpackHINFO, + TypeHIP: unpackHIP, + TypeKEY: unpackKEY, + TypeKX: unpackKX, + TypeL32: unpackL32, + TypeL64: unpackL64, + TypeLOC: unpackLOC, + TypeLP: unpackLP, + TypeMB: unpackMB, + TypeMD: unpackMD, + TypeMF: unpackMF, + TypeMG: unpackMG, + TypeMINFO: unpackMINFO, + TypeMR: unpackMR, + TypeMX: unpackMX, + TypeNAPTR: unpackNAPTR, + TypeNID: unpackNID, + TypeNIMLOC: unpackNIMLOC, + TypeNINFO: unpackNINFO, + TypeNS: unpackNS, + TypeNSAPPTR: unpackNSAPPTR, + TypeNSEC: unpackNSEC, + TypeNSEC3: unpackNSEC3, + TypeNSEC3PARAM: unpackNSEC3PARAM, + TypeOPENPGPKEY: unpackOPENPGPKEY, + TypeOPT: unpackOPT, + TypePTR: unpackPTR, + TypePX: unpackPX, + TypeRKEY: unpackRKEY, + TypeRP: unpackRP, + TypeRRSIG: unpackRRSIG, + TypeRT: unpackRT, + TypeSIG: unpackSIG, + TypeSMIMEA: unpackSMIMEA, + TypeSOA: unpackSOA, + TypeSPF: unpackSPF, + TypeSRV: unpackSRV, + TypeSSHFP: unpackSSHFP, + TypeTA: unpackTA, + TypeTALINK: unpackTALINK, + TypeTKEY: unpackTKEY, + TypeTLSA: unpackTLSA, + TypeTSIG: unpackTSIG, + TypeTXT: unpackTXT, + TypeUID: unpackUID, + TypeUINFO: unpackUINFO, + TypeURI: unpackURI, + TypeX25: unpackX25, +} diff --git a/vendor/github.com/miekg/dns/ztypes.go b/vendor/github.com/miekg/dns/ztypes.go new file mode 100644 index 00000000..3e534f12 --- /dev/null +++ b/vendor/github.com/miekg/dns/ztypes.go @@ -0,0 +1,857 @@ +// *** DO NOT MODIFY *** +// AUTOGENERATED BY go generate from type_generate.go + +package dns + +import ( + "encoding/base64" + "net" +) + +// TypeToRR is a map of constructors for each RR type. +var TypeToRR = map[uint16]func() RR{ + TypeA: func() RR { return new(A) }, + TypeAAAA: func() RR { return new(AAAA) }, + TypeAFSDB: func() RR { return new(AFSDB) }, + TypeANY: func() RR { return new(ANY) }, + TypeAVC: func() RR { return new(AVC) }, + TypeCAA: func() RR { return new(CAA) }, + TypeCDNSKEY: func() RR { return new(CDNSKEY) }, + TypeCDS: func() RR { return new(CDS) }, + TypeCERT: func() RR { return new(CERT) }, + TypeCNAME: func() RR { return new(CNAME) }, + TypeDHCID: func() RR { return new(DHCID) }, + TypeDLV: func() RR { return new(DLV) }, + TypeDNAME: func() RR { return new(DNAME) }, + TypeDNSKEY: func() RR { return new(DNSKEY) }, + TypeDS: func() RR { return new(DS) }, + TypeEID: func() RR { return new(EID) }, + TypeEUI48: func() RR { return new(EUI48) }, + TypeEUI64: func() RR { return new(EUI64) }, + TypeGID: func() RR { return new(GID) }, + TypeGPOS: func() RR { return new(GPOS) }, + TypeHINFO: func() RR { return new(HINFO) }, + TypeHIP: func() RR { return new(HIP) }, + TypeKEY: func() RR { return new(KEY) }, + TypeKX: func() RR { return new(KX) }, + TypeL32: func() RR { return new(L32) }, + TypeL64: func() RR { return new(L64) }, + TypeLOC: func() RR { return new(LOC) }, + TypeLP: func() RR { return new(LP) }, + TypeMB: func() RR { return new(MB) }, + TypeMD: func() RR { return new(MD) }, + TypeMF: func() RR { return new(MF) }, + TypeMG: func() RR { return new(MG) }, + TypeMINFO: func() RR { return new(MINFO) }, + TypeMR: func() RR { return new(MR) }, + TypeMX: func() RR { return new(MX) }, + TypeNAPTR: func() RR { return new(NAPTR) }, + TypeNID: func() RR { return new(NID) }, + TypeNIMLOC: func() RR { return new(NIMLOC) }, + TypeNINFO: func() RR { return new(NINFO) }, + TypeNS: func() RR { return new(NS) }, + TypeNSAPPTR: func() RR { return new(NSAPPTR) }, + TypeNSEC: func() RR { return new(NSEC) }, + TypeNSEC3: func() RR { return new(NSEC3) }, + TypeNSEC3PARAM: func() RR { return new(NSEC3PARAM) }, + TypeOPENPGPKEY: func() RR { return new(OPENPGPKEY) }, + TypeOPT: func() RR { return new(OPT) }, + TypePTR: func() RR { return new(PTR) }, + TypePX: func() RR { return new(PX) }, + TypeRKEY: func() RR { return new(RKEY) }, + TypeRP: func() RR { return new(RP) }, + TypeRRSIG: func() RR { return new(RRSIG) }, + TypeRT: func() RR { return new(RT) }, + TypeSIG: func() RR { return new(SIG) }, + TypeSMIMEA: func() RR { return new(SMIMEA) }, + TypeSOA: func() RR { return new(SOA) }, + TypeSPF: func() RR { return new(SPF) }, + TypeSRV: func() RR { return new(SRV) }, + TypeSSHFP: func() RR { return new(SSHFP) }, + TypeTA: func() RR { return new(TA) }, + TypeTALINK: func() RR { return new(TALINK) }, + TypeTKEY: func() RR { return new(TKEY) }, + TypeTLSA: func() RR { return new(TLSA) }, + TypeTSIG: func() RR { return new(TSIG) }, + TypeTXT: func() RR { return new(TXT) }, + TypeUID: func() RR { return new(UID) }, + TypeUINFO: func() RR { return new(UINFO) }, + TypeURI: func() RR { return new(URI) }, + TypeX25: func() RR { return new(X25) }, +} + +// TypeToString is a map of strings for each RR type. +var TypeToString = map[uint16]string{ + TypeA: "A", + TypeAAAA: "AAAA", + TypeAFSDB: "AFSDB", + TypeANY: "ANY", + TypeATMA: "ATMA", + TypeAVC: "AVC", + TypeAXFR: "AXFR", + TypeCAA: "CAA", + TypeCDNSKEY: "CDNSKEY", + TypeCDS: "CDS", + TypeCERT: "CERT", + TypeCNAME: "CNAME", + TypeDHCID: "DHCID", + TypeDLV: "DLV", + TypeDNAME: "DNAME", + TypeDNSKEY: "DNSKEY", + TypeDS: "DS", + TypeEID: "EID", + TypeEUI48: "EUI48", + TypeEUI64: "EUI64", + TypeGID: "GID", + TypeGPOS: "GPOS", + TypeHINFO: "HINFO", + TypeHIP: "HIP", + TypeISDN: "ISDN", + TypeIXFR: "IXFR", + TypeKEY: "KEY", + TypeKX: "KX", + TypeL32: "L32", + TypeL64: "L64", + TypeLOC: "LOC", + TypeLP: "LP", + TypeMAILA: "MAILA", + TypeMAILB: "MAILB", + TypeMB: "MB", + TypeMD: "MD", + TypeMF: "MF", + TypeMG: "MG", + TypeMINFO: "MINFO", + TypeMR: "MR", + TypeMX: "MX", + TypeNAPTR: "NAPTR", + TypeNID: "NID", + TypeNIMLOC: "NIMLOC", + TypeNINFO: "NINFO", + TypeNS: "NS", + TypeNSEC: "NSEC", + TypeNSEC3: "NSEC3", + TypeNSEC3PARAM: "NSEC3PARAM", + TypeNULL: "NULL", + TypeNXT: "NXT", + TypeNone: "None", + TypeOPENPGPKEY: "OPENPGPKEY", + TypeOPT: "OPT", + TypePTR: "PTR", + TypePX: "PX", + TypeRKEY: "RKEY", + TypeRP: "RP", + TypeRRSIG: "RRSIG", + TypeRT: "RT", + TypeReserved: "Reserved", + TypeSIG: "SIG", + TypeSMIMEA: "SMIMEA", + TypeSOA: "SOA", + TypeSPF: "SPF", + TypeSRV: "SRV", + TypeSSHFP: "SSHFP", + TypeTA: "TA", + TypeTALINK: "TALINK", + TypeTKEY: "TKEY", + TypeTLSA: "TLSA", + TypeTSIG: "TSIG", + TypeTXT: "TXT", + TypeUID: "UID", + TypeUINFO: "UINFO", + TypeUNSPEC: "UNSPEC", + TypeURI: "URI", + TypeX25: "X25", + TypeNSAPPTR: "NSAP-PTR", +} + +// Header() functions +func (rr *A) Header() *RR_Header { return &rr.Hdr } +func (rr *AAAA) Header() *RR_Header { return &rr.Hdr } +func (rr *AFSDB) Header() *RR_Header { return &rr.Hdr } +func (rr *ANY) Header() *RR_Header { return &rr.Hdr } +func (rr *AVC) Header() *RR_Header { return &rr.Hdr } +func (rr *CAA) Header() *RR_Header { return &rr.Hdr } +func (rr *CDNSKEY) Header() *RR_Header { return &rr.Hdr } +func (rr *CDS) Header() *RR_Header { return &rr.Hdr } +func (rr *CERT) Header() *RR_Header { return &rr.Hdr } +func (rr *CNAME) Header() *RR_Header { return &rr.Hdr } +func (rr *DHCID) Header() *RR_Header { return &rr.Hdr } +func (rr *DLV) Header() *RR_Header { return &rr.Hdr } +func (rr *DNAME) Header() *RR_Header { return &rr.Hdr } +func (rr *DNSKEY) Header() *RR_Header { return &rr.Hdr } +func (rr *DS) Header() *RR_Header { return &rr.Hdr } +func (rr *EID) Header() *RR_Header { return &rr.Hdr } +func (rr *EUI48) Header() *RR_Header { return &rr.Hdr } +func (rr *EUI64) Header() *RR_Header { return &rr.Hdr } +func (rr *GID) Header() *RR_Header { return &rr.Hdr } +func (rr *GPOS) Header() *RR_Header { return &rr.Hdr } +func (rr *HINFO) Header() *RR_Header { return &rr.Hdr } +func (rr *HIP) Header() *RR_Header { return &rr.Hdr } +func (rr *KEY) Header() *RR_Header { return &rr.Hdr } +func (rr *KX) Header() *RR_Header { return &rr.Hdr } +func (rr *L32) Header() *RR_Header { return &rr.Hdr } +func (rr *L64) Header() *RR_Header { return &rr.Hdr } +func (rr *LOC) Header() *RR_Header { return &rr.Hdr } +func (rr *LP) Header() *RR_Header { return &rr.Hdr } +func (rr *MB) Header() *RR_Header { return &rr.Hdr } +func (rr *MD) Header() *RR_Header { return &rr.Hdr } +func (rr *MF) Header() *RR_Header { return &rr.Hdr } +func (rr *MG) Header() *RR_Header { return &rr.Hdr } +func (rr *MINFO) Header() *RR_Header { return &rr.Hdr } +func (rr *MR) Header() *RR_Header { return &rr.Hdr } +func (rr *MX) Header() *RR_Header { return &rr.Hdr } +func (rr *NAPTR) Header() *RR_Header { return &rr.Hdr } +func (rr *NID) Header() *RR_Header { return &rr.Hdr } +func (rr *NIMLOC) Header() *RR_Header { return &rr.Hdr } +func (rr *NINFO) Header() *RR_Header { return &rr.Hdr } +func (rr *NS) Header() *RR_Header { return &rr.Hdr } +func (rr *NSAPPTR) Header() *RR_Header { return &rr.Hdr } +func (rr *NSEC) Header() *RR_Header { return &rr.Hdr } +func (rr *NSEC3) Header() *RR_Header { return &rr.Hdr } +func (rr *NSEC3PARAM) Header() *RR_Header { return &rr.Hdr } +func (rr *OPENPGPKEY) Header() *RR_Header { return &rr.Hdr } +func (rr *OPT) Header() *RR_Header { return &rr.Hdr } +func (rr *PTR) Header() *RR_Header { return &rr.Hdr } +func (rr *PX) Header() *RR_Header { return &rr.Hdr } +func (rr *RFC3597) Header() *RR_Header { return &rr.Hdr } +func (rr *RKEY) Header() *RR_Header { return &rr.Hdr } +func (rr *RP) Header() *RR_Header { return &rr.Hdr } +func (rr *RRSIG) Header() *RR_Header { return &rr.Hdr } +func (rr *RT) Header() *RR_Header { return &rr.Hdr } +func (rr *SIG) Header() *RR_Header { return &rr.Hdr } +func (rr *SMIMEA) Header() *RR_Header { return &rr.Hdr } +func (rr *SOA) Header() *RR_Header { return &rr.Hdr } +func (rr *SPF) Header() *RR_Header { return &rr.Hdr } +func (rr *SRV) Header() *RR_Header { return &rr.Hdr } +func (rr *SSHFP) Header() *RR_Header { return &rr.Hdr } +func (rr *TA) Header() *RR_Header { return &rr.Hdr } +func (rr *TALINK) Header() *RR_Header { return &rr.Hdr } +func (rr *TKEY) Header() *RR_Header { return &rr.Hdr } +func (rr *TLSA) Header() *RR_Header { return &rr.Hdr } +func (rr *TSIG) Header() *RR_Header { return &rr.Hdr } +func (rr *TXT) Header() *RR_Header { return &rr.Hdr } +func (rr *UID) Header() *RR_Header { return &rr.Hdr } +func (rr *UINFO) Header() *RR_Header { return &rr.Hdr } +func (rr *URI) Header() *RR_Header { return &rr.Hdr } +func (rr *X25) Header() *RR_Header { return &rr.Hdr } + +// len() functions +func (rr *A) len() int { + l := rr.Hdr.len() + l += net.IPv4len // A + return l +} +func (rr *AAAA) len() int { + l := rr.Hdr.len() + l += net.IPv6len // AAAA + return l +} +func (rr *AFSDB) len() int { + l := rr.Hdr.len() + l += 2 // Subtype + l += len(rr.Hostname) + 1 + return l +} +func (rr *ANY) len() int { + l := rr.Hdr.len() + return l +} +func (rr *AVC) len() int { + l := rr.Hdr.len() + for _, x := range rr.Txt { + l += len(x) + 1 + } + return l +} +func (rr *CAA) len() int { + l := rr.Hdr.len() + l++ // Flag + l += len(rr.Tag) + 1 + l += len(rr.Value) + return l +} +func (rr *CERT) len() int { + l := rr.Hdr.len() + l += 2 // Type + l += 2 // KeyTag + l++ // Algorithm + l += base64.StdEncoding.DecodedLen(len(rr.Certificate)) + return l +} +func (rr *CNAME) len() int { + l := rr.Hdr.len() + l += len(rr.Target) + 1 + return l +} +func (rr *DHCID) len() int { + l := rr.Hdr.len() + l += base64.StdEncoding.DecodedLen(len(rr.Digest)) + return l +} +func (rr *DNAME) len() int { + l := rr.Hdr.len() + l += len(rr.Target) + 1 + return l +} +func (rr *DNSKEY) len() int { + l := rr.Hdr.len() + l += 2 // Flags + l++ // Protocol + l++ // Algorithm + l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) + return l +} +func (rr *DS) len() int { + l := rr.Hdr.len() + l += 2 // KeyTag + l++ // Algorithm + l++ // DigestType + l += len(rr.Digest)/2 + 1 + return l +} +func (rr *EID) len() int { + l := rr.Hdr.len() + l += len(rr.Endpoint)/2 + 1 + return l +} +func (rr *EUI48) len() int { + l := rr.Hdr.len() + l += 6 // Address + return l +} +func (rr *EUI64) len() int { + l := rr.Hdr.len() + l += 8 // Address + return l +} +func (rr *GID) len() int { + l := rr.Hdr.len() + l += 4 // Gid + return l +} +func (rr *GPOS) len() int { + l := rr.Hdr.len() + l += len(rr.Longitude) + 1 + l += len(rr.Latitude) + 1 + l += len(rr.Altitude) + 1 + return l +} +func (rr *HINFO) len() int { + l := rr.Hdr.len() + l += len(rr.Cpu) + 1 + l += len(rr.Os) + 1 + return l +} +func (rr *HIP) len() int { + l := rr.Hdr.len() + l++ // HitLength + l++ // PublicKeyAlgorithm + l += 2 // PublicKeyLength + l += len(rr.Hit)/2 + 1 + l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) + for _, x := range rr.RendezvousServers { + l += len(x) + 1 + } + return l +} +func (rr *KX) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += len(rr.Exchanger) + 1 + return l +} +func (rr *L32) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += net.IPv4len // Locator32 + return l +} +func (rr *L64) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += 8 // Locator64 + return l +} +func (rr *LOC) len() int { + l := rr.Hdr.len() + l++ // Version + l++ // Size + l++ // HorizPre + l++ // VertPre + l += 4 // Latitude + l += 4 // Longitude + l += 4 // Altitude + return l +} +func (rr *LP) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += len(rr.Fqdn) + 1 + return l +} +func (rr *MB) len() int { + l := rr.Hdr.len() + l += len(rr.Mb) + 1 + return l +} +func (rr *MD) len() int { + l := rr.Hdr.len() + l += len(rr.Md) + 1 + return l +} +func (rr *MF) len() int { + l := rr.Hdr.len() + l += len(rr.Mf) + 1 + return l +} +func (rr *MG) len() int { + l := rr.Hdr.len() + l += len(rr.Mg) + 1 + return l +} +func (rr *MINFO) len() int { + l := rr.Hdr.len() + l += len(rr.Rmail) + 1 + l += len(rr.Email) + 1 + return l +} +func (rr *MR) len() int { + l := rr.Hdr.len() + l += len(rr.Mr) + 1 + return l +} +func (rr *MX) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += len(rr.Mx) + 1 + return l +} +func (rr *NAPTR) len() int { + l := rr.Hdr.len() + l += 2 // Order + l += 2 // Preference + l += len(rr.Flags) + 1 + l += len(rr.Service) + 1 + l += len(rr.Regexp) + 1 + l += len(rr.Replacement) + 1 + return l +} +func (rr *NID) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += 8 // NodeID + return l +} +func (rr *NIMLOC) len() int { + l := rr.Hdr.len() + l += len(rr.Locator)/2 + 1 + return l +} +func (rr *NINFO) len() int { + l := rr.Hdr.len() + for _, x := range rr.ZSData { + l += len(x) + 1 + } + return l +} +func (rr *NS) len() int { + l := rr.Hdr.len() + l += len(rr.Ns) + 1 + return l +} +func (rr *NSAPPTR) len() int { + l := rr.Hdr.len() + l += len(rr.Ptr) + 1 + return l +} +func (rr *NSEC3PARAM) len() int { + l := rr.Hdr.len() + l++ // Hash + l++ // Flags + l += 2 // Iterations + l++ // SaltLength + l += len(rr.Salt)/2 + 1 + return l +} +func (rr *OPENPGPKEY) len() int { + l := rr.Hdr.len() + l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) + return l +} +func (rr *PTR) len() int { + l := rr.Hdr.len() + l += len(rr.Ptr) + 1 + return l +} +func (rr *PX) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += len(rr.Map822) + 1 + l += len(rr.Mapx400) + 1 + return l +} +func (rr *RFC3597) len() int { + l := rr.Hdr.len() + l += len(rr.Rdata)/2 + 1 + return l +} +func (rr *RKEY) len() int { + l := rr.Hdr.len() + l += 2 // Flags + l++ // Protocol + l++ // Algorithm + l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) + return l +} +func (rr *RP) len() int { + l := rr.Hdr.len() + l += len(rr.Mbox) + 1 + l += len(rr.Txt) + 1 + return l +} +func (rr *RRSIG) len() int { + l := rr.Hdr.len() + l += 2 // TypeCovered + l++ // Algorithm + l++ // Labels + l += 4 // OrigTtl + l += 4 // Expiration + l += 4 // Inception + l += 2 // KeyTag + l += len(rr.SignerName) + 1 + l += base64.StdEncoding.DecodedLen(len(rr.Signature)) + return l +} +func (rr *RT) len() int { + l := rr.Hdr.len() + l += 2 // Preference + l += len(rr.Host) + 1 + return l +} +func (rr *SMIMEA) len() int { + l := rr.Hdr.len() + l++ // Usage + l++ // Selector + l++ // MatchingType + l += len(rr.Certificate)/2 + 1 + return l +} +func (rr *SOA) len() int { + l := rr.Hdr.len() + l += len(rr.Ns) + 1 + l += len(rr.Mbox) + 1 + l += 4 // Serial + l += 4 // Refresh + l += 4 // Retry + l += 4 // Expire + l += 4 // Minttl + return l +} +func (rr *SPF) len() int { + l := rr.Hdr.len() + for _, x := range rr.Txt { + l += len(x) + 1 + } + return l +} +func (rr *SRV) len() int { + l := rr.Hdr.len() + l += 2 // Priority + l += 2 // Weight + l += 2 // Port + l += len(rr.Target) + 1 + return l +} +func (rr *SSHFP) len() int { + l := rr.Hdr.len() + l++ // Algorithm + l++ // Type + l += len(rr.FingerPrint)/2 + 1 + return l +} +func (rr *TA) len() int { + l := rr.Hdr.len() + l += 2 // KeyTag + l++ // Algorithm + l++ // DigestType + l += len(rr.Digest)/2 + 1 + return l +} +func (rr *TALINK) len() int { + l := rr.Hdr.len() + l += len(rr.PreviousName) + 1 + l += len(rr.NextName) + 1 + return l +} +func (rr *TKEY) len() int { + l := rr.Hdr.len() + l += len(rr.Algorithm) + 1 + l += 4 // Inception + l += 4 // Expiration + l += 2 // Mode + l += 2 // Error + l += 2 // KeySize + l += len(rr.Key) + 1 + l += 2 // OtherLen + l += len(rr.OtherData) + 1 + return l +} +func (rr *TLSA) len() int { + l := rr.Hdr.len() + l++ // Usage + l++ // Selector + l++ // MatchingType + l += len(rr.Certificate)/2 + 1 + return l +} +func (rr *TSIG) len() int { + l := rr.Hdr.len() + l += len(rr.Algorithm) + 1 + l += 6 // TimeSigned + l += 2 // Fudge + l += 2 // MACSize + l += len(rr.MAC)/2 + 1 + l += 2 // OrigId + l += 2 // Error + l += 2 // OtherLen + l += len(rr.OtherData)/2 + 1 + return l +} +func (rr *TXT) len() int { + l := rr.Hdr.len() + for _, x := range rr.Txt { + l += len(x) + 1 + } + return l +} +func (rr *UID) len() int { + l := rr.Hdr.len() + l += 4 // Uid + return l +} +func (rr *UINFO) len() int { + l := rr.Hdr.len() + l += len(rr.Uinfo) + 1 + return l +} +func (rr *URI) len() int { + l := rr.Hdr.len() + l += 2 // Priority + l += 2 // Weight + l += len(rr.Target) + return l +} +func (rr *X25) len() int { + l := rr.Hdr.len() + l += len(rr.PSDNAddress) + 1 + return l +} + +// copy() functions +func (rr *A) copy() RR { + return &A{*rr.Hdr.copyHeader(), copyIP(rr.A)} +} +func (rr *AAAA) copy() RR { + return &AAAA{*rr.Hdr.copyHeader(), copyIP(rr.AAAA)} +} +func (rr *AFSDB) copy() RR { + return &AFSDB{*rr.Hdr.copyHeader(), rr.Subtype, rr.Hostname} +} +func (rr *ANY) copy() RR { + return &ANY{*rr.Hdr.copyHeader()} +} +func (rr *AVC) copy() RR { + Txt := make([]string, len(rr.Txt)) + copy(Txt, rr.Txt) + return &AVC{*rr.Hdr.copyHeader(), Txt} +} +func (rr *CAA) copy() RR { + return &CAA{*rr.Hdr.copyHeader(), rr.Flag, rr.Tag, rr.Value} +} +func (rr *CERT) copy() RR { + return &CERT{*rr.Hdr.copyHeader(), rr.Type, rr.KeyTag, rr.Algorithm, rr.Certificate} +} +func (rr *CNAME) copy() RR { + return &CNAME{*rr.Hdr.copyHeader(), rr.Target} +} +func (rr *DHCID) copy() RR { + return &DHCID{*rr.Hdr.copyHeader(), rr.Digest} +} +func (rr *DNAME) copy() RR { + return &DNAME{*rr.Hdr.copyHeader(), rr.Target} +} +func (rr *DNSKEY) copy() RR { + return &DNSKEY{*rr.Hdr.copyHeader(), rr.Flags, rr.Protocol, rr.Algorithm, rr.PublicKey} +} +func (rr *DS) copy() RR { + return &DS{*rr.Hdr.copyHeader(), rr.KeyTag, rr.Algorithm, rr.DigestType, rr.Digest} +} +func (rr *EID) copy() RR { + return &EID{*rr.Hdr.copyHeader(), rr.Endpoint} +} +func (rr *EUI48) copy() RR { + return &EUI48{*rr.Hdr.copyHeader(), rr.Address} +} +func (rr *EUI64) copy() RR { + return &EUI64{*rr.Hdr.copyHeader(), rr.Address} +} +func (rr *GID) copy() RR { + return &GID{*rr.Hdr.copyHeader(), rr.Gid} +} +func (rr *GPOS) copy() RR { + return &GPOS{*rr.Hdr.copyHeader(), rr.Longitude, rr.Latitude, rr.Altitude} +} +func (rr *HINFO) copy() RR { + return &HINFO{*rr.Hdr.copyHeader(), rr.Cpu, rr.Os} +} +func (rr *HIP) copy() RR { + RendezvousServers := make([]string, len(rr.RendezvousServers)) + copy(RendezvousServers, rr.RendezvousServers) + return &HIP{*rr.Hdr.copyHeader(), rr.HitLength, rr.PublicKeyAlgorithm, rr.PublicKeyLength, rr.Hit, rr.PublicKey, RendezvousServers} +} +func (rr *KX) copy() RR { + return &KX{*rr.Hdr.copyHeader(), rr.Preference, rr.Exchanger} +} +func (rr *L32) copy() RR { + return &L32{*rr.Hdr.copyHeader(), rr.Preference, copyIP(rr.Locator32)} +} +func (rr *L64) copy() RR { + return &L64{*rr.Hdr.copyHeader(), rr.Preference, rr.Locator64} +} +func (rr *LOC) copy() RR { + return &LOC{*rr.Hdr.copyHeader(), rr.Version, rr.Size, rr.HorizPre, rr.VertPre, rr.Latitude, rr.Longitude, rr.Altitude} +} +func (rr *LP) copy() RR { + return &LP{*rr.Hdr.copyHeader(), rr.Preference, rr.Fqdn} +} +func (rr *MB) copy() RR { + return &MB{*rr.Hdr.copyHeader(), rr.Mb} +} +func (rr *MD) copy() RR { + return &MD{*rr.Hdr.copyHeader(), rr.Md} +} +func (rr *MF) copy() RR { + return &MF{*rr.Hdr.copyHeader(), rr.Mf} +} +func (rr *MG) copy() RR { + return &MG{*rr.Hdr.copyHeader(), rr.Mg} +} +func (rr *MINFO) copy() RR { + return &MINFO{*rr.Hdr.copyHeader(), rr.Rmail, rr.Email} +} +func (rr *MR) copy() RR { + return &MR{*rr.Hdr.copyHeader(), rr.Mr} +} +func (rr *MX) copy() RR { + return &MX{*rr.Hdr.copyHeader(), rr.Preference, rr.Mx} +} +func (rr *NAPTR) copy() RR { + return &NAPTR{*rr.Hdr.copyHeader(), rr.Order, rr.Preference, rr.Flags, rr.Service, rr.Regexp, rr.Replacement} +} +func (rr *NID) copy() RR { + return &NID{*rr.Hdr.copyHeader(), rr.Preference, rr.NodeID} +} +func (rr *NIMLOC) copy() RR { + return &NIMLOC{*rr.Hdr.copyHeader(), rr.Locator} +} +func (rr *NINFO) copy() RR { + ZSData := make([]string, len(rr.ZSData)) + copy(ZSData, rr.ZSData) + return &NINFO{*rr.Hdr.copyHeader(), ZSData} +} +func (rr *NS) copy() RR { + return &NS{*rr.Hdr.copyHeader(), rr.Ns} +} +func (rr *NSAPPTR) copy() RR { + return &NSAPPTR{*rr.Hdr.copyHeader(), rr.Ptr} +} +func (rr *NSEC) copy() RR { + TypeBitMap := make([]uint16, len(rr.TypeBitMap)) + copy(TypeBitMap, rr.TypeBitMap) + return &NSEC{*rr.Hdr.copyHeader(), rr.NextDomain, TypeBitMap} +} +func (rr *NSEC3) copy() RR { + TypeBitMap := make([]uint16, len(rr.TypeBitMap)) + copy(TypeBitMap, rr.TypeBitMap) + return &NSEC3{*rr.Hdr.copyHeader(), rr.Hash, rr.Flags, rr.Iterations, rr.SaltLength, rr.Salt, rr.HashLength, rr.NextDomain, TypeBitMap} +} +func (rr *NSEC3PARAM) copy() RR { + return &NSEC3PARAM{*rr.Hdr.copyHeader(), rr.Hash, rr.Flags, rr.Iterations, rr.SaltLength, rr.Salt} +} +func (rr *OPENPGPKEY) copy() RR { + return &OPENPGPKEY{*rr.Hdr.copyHeader(), rr.PublicKey} +} +func (rr *OPT) copy() RR { + Option := make([]EDNS0, len(rr.Option)) + copy(Option, rr.Option) + return &OPT{*rr.Hdr.copyHeader(), Option} +} +func (rr *PTR) copy() RR { + return &PTR{*rr.Hdr.copyHeader(), rr.Ptr} +} +func (rr *PX) copy() RR { + return &PX{*rr.Hdr.copyHeader(), rr.Preference, rr.Map822, rr.Mapx400} +} +func (rr *RFC3597) copy() RR { + return &RFC3597{*rr.Hdr.copyHeader(), rr.Rdata} +} +func (rr *RKEY) copy() RR { + return &RKEY{*rr.Hdr.copyHeader(), rr.Flags, rr.Protocol, rr.Algorithm, rr.PublicKey} +} +func (rr *RP) copy() RR { + return &RP{*rr.Hdr.copyHeader(), rr.Mbox, rr.Txt} +} +func (rr *RRSIG) copy() RR { + return &RRSIG{*rr.Hdr.copyHeader(), rr.TypeCovered, rr.Algorithm, rr.Labels, rr.OrigTtl, rr.Expiration, rr.Inception, rr.KeyTag, rr.SignerName, rr.Signature} +} +func (rr *RT) copy() RR { + return &RT{*rr.Hdr.copyHeader(), rr.Preference, rr.Host} +} +func (rr *SMIMEA) copy() RR { + return &SMIMEA{*rr.Hdr.copyHeader(), rr.Usage, rr.Selector, rr.MatchingType, rr.Certificate} +} +func (rr *SOA) copy() RR { + return &SOA{*rr.Hdr.copyHeader(), rr.Ns, rr.Mbox, rr.Serial, rr.Refresh, rr.Retry, rr.Expire, rr.Minttl} +} +func (rr *SPF) copy() RR { + Txt := make([]string, len(rr.Txt)) + copy(Txt, rr.Txt) + return &SPF{*rr.Hdr.copyHeader(), Txt} +} +func (rr *SRV) copy() RR { + return &SRV{*rr.Hdr.copyHeader(), rr.Priority, rr.Weight, rr.Port, rr.Target} +} +func (rr *SSHFP) copy() RR { + return &SSHFP{*rr.Hdr.copyHeader(), rr.Algorithm, rr.Type, rr.FingerPrint} +} +func (rr *TA) copy() RR { + return &TA{*rr.Hdr.copyHeader(), rr.KeyTag, rr.Algorithm, rr.DigestType, rr.Digest} +} +func (rr *TALINK) copy() RR { + return &TALINK{*rr.Hdr.copyHeader(), rr.PreviousName, rr.NextName} +} +func (rr *TKEY) copy() RR { + return &TKEY{*rr.Hdr.copyHeader(), rr.Algorithm, rr.Inception, rr.Expiration, rr.Mode, rr.Error, rr.KeySize, rr.Key, rr.OtherLen, rr.OtherData} +} +func (rr *TLSA) copy() RR { + return &TLSA{*rr.Hdr.copyHeader(), rr.Usage, rr.Selector, rr.MatchingType, rr.Certificate} +} +func (rr *TSIG) copy() RR { + return &TSIG{*rr.Hdr.copyHeader(), rr.Algorithm, rr.TimeSigned, rr.Fudge, rr.MACSize, rr.MAC, rr.OrigId, rr.Error, rr.OtherLen, rr.OtherData} +} +func (rr *TXT) copy() RR { + Txt := make([]string, len(rr.Txt)) + copy(Txt, rr.Txt) + return &TXT{*rr.Hdr.copyHeader(), Txt} +} +func (rr *UID) copy() RR { + return &UID{*rr.Hdr.copyHeader(), rr.Uid} +} +func (rr *UINFO) copy() RR { + return &UINFO{*rr.Hdr.copyHeader(), rr.Uinfo} +} +func (rr *URI) copy() RR { + return &URI{*rr.Hdr.copyHeader(), rr.Priority, rr.Weight, rr.Target} +} +func (rr *X25) copy() RR { + return &X25{*rr.Hdr.copyHeader(), rr.PSDNAddress} +} diff --git a/vendor/github.com/mitchellh/go-homedir/homedir.go b/vendor/github.com/mitchellh/go-homedir/homedir.go index 6944957d..47e1f9ef 100644 --- a/vendor/github.com/mitchellh/go-homedir/homedir.go +++ b/vendor/github.com/mitchellh/go-homedir/homedir.go @@ -87,7 +87,7 @@ func dirUnix() (string, error) { cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid())) cmd.Stdout = &stdout if err := cmd.Run(); err != nil { - // If "getent" is missing, ignore it + // If the error is ErrNotFound, we ignore it. Otherwise, return it. if err != exec.ErrNotFound { return "", err } @@ -118,6 +118,11 @@ func dirUnix() (string, error) { } func dirWindows() (string, error) { + // First prefer the HOME environmental variable + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + drive := os.Getenv("HOMEDRIVE") path := os.Getenv("HOMEPATH") home := drive + path diff --git a/vendor/github.com/mitchellh/go-homedir/homedir_test.go b/vendor/github.com/mitchellh/go-homedir/homedir_test.go deleted file mode 100644 index c34dbc7f..00000000 --- a/vendor/github.com/mitchellh/go-homedir/homedir_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package homedir - -import ( - "fmt" - "os" - "os/user" - "testing" -) - -func patchEnv(key, value string) func() { - bck := os.Getenv(key) - deferFunc := func() { - os.Setenv(key, bck) - } - - os.Setenv(key, value) - return deferFunc -} - -func BenchmarkDir(b *testing.B) { - // We do this for any "warmups" - for i := 0; i < 10; i++ { - Dir() - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - Dir() - } -} - -func TestDir(t *testing.T) { - u, err := user.Current() - if err != nil { - t.Fatalf("err: %s", err) - } - - dir, err := Dir() - if err != nil { - t.Fatalf("err: %s", err) - } - - if u.HomeDir != dir { - t.Fatalf("%#v != %#v", u.HomeDir, dir) - } -} - -func TestExpand(t *testing.T) { - u, err := user.Current() - if err != nil { - t.Fatalf("err: %s", err) - } - - cases := []struct { - Input string - Output string - Err bool - }{ - { - "/foo", - "/foo", - false, - }, - - { - "~/foo", - fmt.Sprintf("%s/foo", u.HomeDir), - false, - }, - - { - "", - "", - false, - }, - - { - "~", - u.HomeDir, - false, - }, - - { - "~foo/foo", - "", - true, - }, - } - - for _, tc := range cases { - actual, err := Expand(tc.Input) - if (err != nil) != tc.Err { - t.Fatalf("Input: %#v\n\nErr: %s", tc.Input, err) - } - - if actual != tc.Output { - t.Fatalf("Input: %#v\n\nOutput: %#v", tc.Input, actual) - } - } - - DisableCache = true - defer func() { DisableCache = false }() - defer patchEnv("HOME", "/custom/path/")() - expected := "/custom/path/foo/bar" - actual, err := Expand("~/foo/bar") - - if err != nil { - t.Errorf("No error is expected, got: %v", err) - } else if actual != "/custom/path/foo/bar" { - t.Errorf("Expected: %v; actual: %v", expected, actual) - } -} diff --git a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go index aa91f76c..115ae67c 100644 --- a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go +++ b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go @@ -72,7 +72,10 @@ func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc { } // Modify the from kind to be correct with the new data - f = reflect.ValueOf(data).Type() + f = nil + if val := reflect.ValueOf(data); val.IsValid() { + f = val.Type() + } } return data, nil diff --git a/vendor/github.com/mitchellh/mapstructure/decode_hooks_test.go b/vendor/github.com/mitchellh/mapstructure/decode_hooks_test.go deleted file mode 100644 index 53289afc..00000000 --- a/vendor/github.com/mitchellh/mapstructure/decode_hooks_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package mapstructure - -import ( - "errors" - "reflect" - "testing" - "time" -) - -func TestComposeDecodeHookFunc(t *testing.T) { - f1 := func( - f reflect.Kind, - t reflect.Kind, - data interface{}) (interface{}, error) { - return data.(string) + "foo", nil - } - - f2 := func( - f reflect.Kind, - t reflect.Kind, - data interface{}) (interface{}, error) { - return data.(string) + "bar", nil - } - - f := ComposeDecodeHookFunc(f1, f2) - - result, err := DecodeHookExec( - f, reflect.TypeOf(""), reflect.TypeOf([]byte("")), "") - if err != nil { - t.Fatalf("bad: %s", err) - } - if result.(string) != "foobar" { - t.Fatalf("bad: %#v", result) - } -} - -func TestComposeDecodeHookFunc_err(t *testing.T) { - f1 := func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) { - return nil, errors.New("foo") - } - - f2 := func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) { - panic("NOPE") - } - - f := ComposeDecodeHookFunc(f1, f2) - - _, err := DecodeHookExec( - f, reflect.TypeOf(""), reflect.TypeOf([]byte("")), 42) - if err.Error() != "foo" { - t.Fatalf("bad: %s", err) - } -} - -func TestComposeDecodeHookFunc_kinds(t *testing.T) { - var f2From reflect.Kind - - f1 := func( - f reflect.Kind, - t reflect.Kind, - data interface{}) (interface{}, error) { - return int(42), nil - } - - f2 := func( - f reflect.Kind, - t reflect.Kind, - data interface{}) (interface{}, error) { - f2From = f - return data, nil - } - - f := ComposeDecodeHookFunc(f1, f2) - - _, err := DecodeHookExec( - f, reflect.TypeOf(""), reflect.TypeOf([]byte("")), "") - if err != nil { - t.Fatalf("bad: %s", err) - } - if f2From != reflect.Int { - t.Fatalf("bad: %#v", f2From) - } -} - -func TestStringToSliceHookFunc(t *testing.T) { - f := StringToSliceHookFunc(",") - - strType := reflect.TypeOf("") - sliceType := reflect.TypeOf([]byte("")) - cases := []struct { - f, t reflect.Type - data interface{} - result interface{} - err bool - }{ - {sliceType, sliceType, 42, 42, false}, - {strType, strType, 42, 42, false}, - { - strType, - sliceType, - "foo,bar,baz", - []string{"foo", "bar", "baz"}, - false, - }, - { - strType, - sliceType, - "", - []string{}, - false, - }, - } - - for i, tc := range cases { - actual, err := DecodeHookExec(f, tc.f, tc.t, tc.data) - if tc.err != (err != nil) { - t.Fatalf("case %d: expected err %#v", i, tc.err) - } - if !reflect.DeepEqual(actual, tc.result) { - t.Fatalf( - "case %d: expected %#v, got %#v", - i, tc.result, actual) - } - } -} - -func TestStringToTimeDurationHookFunc(t *testing.T) { - f := StringToTimeDurationHookFunc() - - strType := reflect.TypeOf("") - timeType := reflect.TypeOf(time.Duration(5)) - cases := []struct { - f, t reflect.Type - data interface{} - result interface{} - err bool - }{ - {strType, timeType, "5s", 5 * time.Second, false}, - {strType, timeType, "5", time.Duration(0), true}, - {strType, strType, "5", "5", false}, - } - - for i, tc := range cases { - actual, err := DecodeHookExec(f, tc.f, tc.t, tc.data) - if tc.err != (err != nil) { - t.Fatalf("case %d: expected err %#v", i, tc.err) - } - if !reflect.DeepEqual(actual, tc.result) { - t.Fatalf( - "case %d: expected %#v, got %#v", - i, tc.result, actual) - } - } -} - -func TestWeaklyTypedHook(t *testing.T) { - var f DecodeHookFunc = WeaklyTypedHook - - boolType := reflect.TypeOf(true) - strType := reflect.TypeOf("") - sliceType := reflect.TypeOf([]byte("")) - cases := []struct { - f, t reflect.Type - data interface{} - result interface{} - err bool - }{ - // TO STRING - { - boolType, - strType, - false, - "0", - false, - }, - - { - boolType, - strType, - true, - "1", - false, - }, - - { - reflect.TypeOf(float32(1)), - strType, - float32(7), - "7", - false, - }, - - { - reflect.TypeOf(int(1)), - strType, - int(7), - "7", - false, - }, - - { - sliceType, - strType, - []uint8("foo"), - "foo", - false, - }, - - { - reflect.TypeOf(uint(1)), - strType, - uint(7), - "7", - false, - }, - } - - for i, tc := range cases { - actual, err := DecodeHookExec(f, tc.f, tc.t, tc.data) - if tc.err != (err != nil) { - t.Fatalf("case %d: expected err %#v", i, tc.err) - } - if !reflect.DeepEqual(actual, tc.result) { - t.Fatalf( - "case %d: expected %#v, got %#v", - i, tc.result, actual) - } - } -} diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure.go b/vendor/github.com/mitchellh/mapstructure/mapstructure.go index a367a95b..6dee0ef0 100644 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure.go +++ b/vendor/github.com/mitchellh/mapstructure/mapstructure.go @@ -1,5 +1,5 @@ // The mapstructure package exposes functionality to convert an -// abitrary map[string]interface{} into a native Go structure. +// arbitrary map[string]interface{} into a native Go structure. // // The Go structure can be arbitrarily complex, containing slices, // other structs, etc. and the decoder will properly decode nested @@ -8,6 +8,7 @@ package mapstructure import ( + "encoding/json" "errors" "fmt" "reflect" @@ -68,6 +69,9 @@ type DecoderConfig struct { // - empty array = empty map and vice versa // - negative numbers to overflowed uint values (base 10) // - slice of maps to a merged map + // - single values are converted to slices if required. Each + // element is weakly decoded. For example: "4" can become []int{4} + // if the target type is an int slice. // WeaklyTypedInput bool @@ -201,7 +205,7 @@ func (d *Decoder) decode(name string, data interface{}, val reflect.Value) error d.config.DecodeHook, dataVal.Type(), val.Type(), data) if err != nil { - return err + return fmt.Errorf("error decoding '%s': %s", name, err) } } @@ -228,6 +232,8 @@ func (d *Decoder) decode(name string, data interface{}, val reflect.Value) error err = d.decodePtr(name, data, val) case reflect.Slice: err = d.decodeSlice(name, data, val) + case reflect.Func: + err = d.decodeFunc(name, data, val) default: // If we reached this point then we weren't able to decode it return fmt.Errorf("%s: unsupported type: %s", name, dataKind) @@ -246,6 +252,10 @@ func (d *Decoder) decode(name string, data interface{}, val reflect.Value) error // value to "data" of that type. func (d *Decoder) decodeBasic(name string, data interface{}, val reflect.Value) error { dataVal := reflect.ValueOf(data) + if !dataVal.IsValid() { + dataVal = reflect.Zero(val.Type()) + } + dataValType := dataVal.Type() if !dataValType.AssignableTo(val.Type()) { return fmt.Errorf( @@ -302,6 +312,7 @@ func (d *Decoder) decodeString(name string, data interface{}, val reflect.Value) func (d *Decoder) decodeInt(name string, data interface{}, val reflect.Value) error { dataVal := reflect.ValueOf(data) dataKind := getKind(dataVal) + dataType := dataVal.Type() switch { case dataKind == reflect.Int: @@ -323,6 +334,14 @@ func (d *Decoder) decodeInt(name string, data interface{}, val reflect.Value) er } else { return fmt.Errorf("cannot parse '%s' as int: %s", name, err) } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Int64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + val.SetInt(i) default: return fmt.Errorf( "'%s' expected type '%s', got unconvertible type '%s'", @@ -409,6 +428,7 @@ func (d *Decoder) decodeBool(name string, data interface{}, val reflect.Value) e func (d *Decoder) decodeFloat(name string, data interface{}, val reflect.Value) error { dataVal := reflect.ValueOf(data) dataKind := getKind(dataVal) + dataType := dataVal.Type() switch { case dataKind == reflect.Int: @@ -430,6 +450,14 @@ func (d *Decoder) decodeFloat(name string, data interface{}, val reflect.Value) } else { return fmt.Errorf("cannot parse '%s' as float: %s", name, err) } + case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": + jn := data.(json.Number) + i, err := jn.Float64() + if err != nil { + return fmt.Errorf( + "error decoding json.Number into %s: %s", name, err) + } + val.SetFloat(i) default: return fmt.Errorf( "'%s' expected type '%s', got unconvertible type '%s'", @@ -523,7 +551,12 @@ func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) er // into that. Then set the value of the pointer to this type. valType := val.Type() valElemType := valType.Elem() - realVal := reflect.New(valElemType) + + realVal := val + if realVal.IsNil() || d.config.ZeroFields { + realVal = reflect.New(valElemType) + } + if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil { return err } @@ -532,6 +565,19 @@ func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) er return nil } +func (d *Decoder) decodeFunc(name string, data interface{}, val reflect.Value) error { + // Create an element of the concrete (non pointer) type and decode + // into that. Then set the value of the pointer to this type. + dataVal := reflect.Indirect(reflect.ValueOf(data)) + if val.Type() != dataVal.Type() { + return fmt.Errorf( + "'%s' expected type '%s', got unconvertible type '%s'", + name, val.Type(), dataVal.Type()) + } + val.Set(dataVal) + return nil +} + func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) error { dataVal := reflect.Indirect(reflect.ValueOf(data)) dataValKind := dataVal.Kind() @@ -539,26 +585,44 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) valElemType := valType.Elem() sliceType := reflect.SliceOf(valElemType) - // Check input type - if dataValKind != reflect.Array && dataValKind != reflect.Slice { - // Accept empty map instead of array/slice in weakly typed mode - if d.config.WeaklyTypedInput && dataVal.Kind() == reflect.Map && dataVal.Len() == 0 { - val.Set(reflect.MakeSlice(sliceType, 0, 0)) - return nil - } else { + valSlice := val + if valSlice.IsNil() || d.config.ZeroFields { + // Check input type + if dataValKind != reflect.Array && dataValKind != reflect.Slice { + if d.config.WeaklyTypedInput { + switch { + // Empty maps turn into empty slices + case dataValKind == reflect.Map: + if dataVal.Len() == 0 { + val.Set(reflect.MakeSlice(sliceType, 0, 0)) + return nil + } + + // All other types we try to convert to the slice type + // and "lift" it into it. i.e. a string becomes a string slice. + default: + // Just re-try this function with data as a slice. + return d.decodeSlice(name, []interface{}{data}, val) + } + } + return fmt.Errorf( "'%s': source data must be an array or slice, got %s", name, dataValKind) - } - } - // Make a new slice to hold our result, same size as the original data. - valSlice := reflect.MakeSlice(sliceType, dataVal.Len(), dataVal.Len()) + } + + // Make a new slice to hold our result, same size as the original data. + valSlice = reflect.MakeSlice(sliceType, dataVal.Len(), dataVal.Len()) + } // Accumulate any errors errors := make([]string, 0) for i := 0; i < dataVal.Len(); i++ { currentData := dataVal.Index(i).Interface() + for valSlice.Len() <= i { + valSlice = reflect.Append(valSlice, reflect.Zero(valElemType)) + } currentField := valSlice.Index(i) fieldName := fmt.Sprintf("%s[%d]", name, i) @@ -628,14 +692,6 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) fieldType := structType.Field(i) fieldKind := fieldType.Type.Kind() - if fieldType.Anonymous { - if fieldKind != reflect.Struct { - errors = appendErrors(errors, - fmt.Errorf("%s: unsupported type: %s", fieldType.Name, fieldKind)) - continue - } - } - // If "squash" is specified in the tag, we squash the field down. squash := false tagParts := strings.Split(fieldType.Tag.Get(d.config.TagName), ",") @@ -675,7 +731,7 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) if !rawMapVal.IsValid() { // Do a slower search by iterating over each key and // doing case-insensitive search. - for dataValKey, _ := range dataValKeys { + for dataValKey := range dataValKeys { mK, ok := dataValKey.Interface().(string) if !ok { // Not a string key @@ -723,7 +779,7 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) if d.config.ErrorUnused && len(dataValKeysUnused) > 0 { keys := make([]string, 0, len(dataValKeysUnused)) - for rawKey, _ := range dataValKeysUnused { + for rawKey := range dataValKeysUnused { keys = append(keys, rawKey.(string)) } sort.Strings(keys) @@ -738,7 +794,7 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) // Add the unused keys to the list of unused keys if we're tracking metadata if d.config.Metadata != nil { - for rawKey, _ := range dataValKeysUnused { + for rawKey := range dataValKeysUnused { key := rawKey.(string) if name != "" { key = fmt.Sprintf("%s.%s", name, key) diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure_benchmark_test.go b/vendor/github.com/mitchellh/mapstructure/mapstructure_benchmark_test.go deleted file mode 100644 index 41d2a41f..00000000 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure_benchmark_test.go +++ /dev/null @@ -1,279 +0,0 @@ -package mapstructure - -import ( - "encoding/json" - "testing" -) - -func Benchmark_Decode(b *testing.B) { - type Person struct { - Name string - Age int - Emails []string - Extra map[string]string - } - - input := map[string]interface{}{ - "name": "Mitchell", - "age": 91, - "emails": []string{"one", "two", "three"}, - "extra": map[string]string{ - "twitter": "mitchellh", - }, - } - - var result Person - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} - -// decodeViaJSON takes the map data and passes it through encoding/json to convert it into the -// given Go native structure pointed to by v. v must be a pointer to a struct. -func decodeViaJSON(data interface{}, v interface{}) error { - // Perform the task by simply marshalling the input into JSON, - // then unmarshalling it into target native Go struct. - b, err := json.Marshal(data) - if err != nil { - return err - } - return json.Unmarshal(b, v) -} - -func Benchmark_DecodeViaJSON(b *testing.B) { - type Person struct { - Name string - Age int - Emails []string - Extra map[string]string - } - - input := map[string]interface{}{ - "name": "Mitchell", - "age": 91, - "emails": []string{"one", "two", "three"}, - "extra": map[string]string{ - "twitter": "mitchellh", - }, - } - - var result Person - for i := 0; i < b.N; i++ { - decodeViaJSON(input, &result) - } -} - -func Benchmark_DecodeBasic(b *testing.B) { - input := map[string]interface{}{ - "vstring": "foo", - "vint": 42, - "Vuint": 42, - "vbool": true, - "Vfloat": 42.42, - "vsilent": true, - "vdata": 42, - } - - var result Basic - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} - -func Benchmark_DecodeEmbedded(b *testing.B) { - input := map[string]interface{}{ - "vstring": "foo", - "Basic": map[string]interface{}{ - "vstring": "innerfoo", - }, - "vunique": "bar", - } - - var result Embedded - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} - -func Benchmark_DecodeTypeConversion(b *testing.B) { - input := map[string]interface{}{ - "IntToFloat": 42, - "IntToUint": 42, - "IntToBool": 1, - "IntToString": 42, - "UintToInt": 42, - "UintToFloat": 42, - "UintToBool": 42, - "UintToString": 42, - "BoolToInt": true, - "BoolToUint": true, - "BoolToFloat": true, - "BoolToString": true, - "FloatToInt": 42.42, - "FloatToUint": 42.42, - "FloatToBool": 42.42, - "FloatToString": 42.42, - "StringToInt": "42", - "StringToUint": "42", - "StringToBool": "1", - "StringToFloat": "42.42", - "SliceToMap": []interface{}{}, - "MapToSlice": map[string]interface{}{}, - } - - var resultStrict TypeConversionResult - for i := 0; i < b.N; i++ { - Decode(input, &resultStrict) - } -} - -func Benchmark_DecodeMap(b *testing.B) { - input := map[string]interface{}{ - "vfoo": "foo", - "vother": map[interface{}]interface{}{ - "foo": "foo", - "bar": "bar", - }, - } - - var result Map - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} - -func Benchmark_DecodeMapOfStruct(b *testing.B) { - input := map[string]interface{}{ - "value": map[string]interface{}{ - "foo": map[string]string{"vstring": "one"}, - "bar": map[string]string{"vstring": "two"}, - }, - } - - var result MapOfStruct - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} - -func Benchmark_DecodeSlice(b *testing.B) { - input := map[string]interface{}{ - "vfoo": "foo", - "vbar": []string{"foo", "bar", "baz"}, - } - - var result Slice - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} - -func Benchmark_DecodeSliceOfStruct(b *testing.B) { - input := map[string]interface{}{ - "value": []map[string]interface{}{ - {"vstring": "one"}, - {"vstring": "two"}, - }, - } - - var result SliceOfStruct - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} - -func Benchmark_DecodeWeaklyTypedInput(b *testing.B) { - type Person struct { - Name string - Age int - Emails []string - } - - // This input can come from anywhere, but typically comes from - // something like decoding JSON, generated by a weakly typed language - // such as PHP. - input := map[string]interface{}{ - "name": 123, // number => string - "age": "42", // string => number - "emails": map[string]interface{}{}, // empty map => empty array - } - - var result Person - config := &DecoderConfig{ - WeaklyTypedInput: true, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - panic(err) - } - - for i := 0; i < b.N; i++ { - decoder.Decode(input) - } -} - -func Benchmark_DecodeMetadata(b *testing.B) { - type Person struct { - Name string - Age int - } - - input := map[string]interface{}{ - "name": "Mitchell", - "age": 91, - "email": "foo@bar.com", - } - - var md Metadata - var result Person - config := &DecoderConfig{ - Metadata: &md, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - panic(err) - } - - for i := 0; i < b.N; i++ { - decoder.Decode(input) - } -} - -func Benchmark_DecodeMetadataEmbedded(b *testing.B) { - input := map[string]interface{}{ - "vstring": "foo", - "vunique": "bar", - } - - var md Metadata - var result EmbeddedSquash - config := &DecoderConfig{ - Metadata: &md, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - b.Fatalf("err: %s", err) - } - - for i := 0; i < b.N; i++ { - decoder.Decode(input) - } -} - -func Benchmark_DecodeTagged(b *testing.B) { - input := map[string]interface{}{ - "foo": "bar", - "bar": "value", - } - - var result Tagged - for i := 0; i < b.N; i++ { - Decode(input, &result) - } -} diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure_bugs_test.go b/vendor/github.com/mitchellh/mapstructure/mapstructure_bugs_test.go deleted file mode 100644 index 7054f1ac..00000000 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure_bugs_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package mapstructure - -import "testing" - -// GH-1 -func TestDecode_NilValue(t *testing.T) { - input := map[string]interface{}{ - "vfoo": nil, - "vother": nil, - } - - var result Map - err := Decode(input, &result) - if err != nil { - t.Fatalf("should not error: %s", err) - } - - if result.Vfoo != "" { - t.Fatalf("value should be default: %s", result.Vfoo) - } - - if result.Vother != nil { - t.Fatalf("Vother should be nil: %s", result.Vother) - } -} - -// GH-10 -func TestDecode_mapInterfaceInterface(t *testing.T) { - input := map[interface{}]interface{}{ - "vfoo": nil, - "vother": nil, - } - - var result Map - err := Decode(input, &result) - if err != nil { - t.Fatalf("should not error: %s", err) - } - - if result.Vfoo != "" { - t.Fatalf("value should be default: %s", result.Vfoo) - } - - if result.Vother != nil { - t.Fatalf("Vother should be nil: %s", result.Vother) - } -} diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure_examples_test.go b/vendor/github.com/mitchellh/mapstructure/mapstructure_examples_test.go deleted file mode 100644 index f17c214a..00000000 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure_examples_test.go +++ /dev/null @@ -1,203 +0,0 @@ -package mapstructure - -import ( - "fmt" -) - -func ExampleDecode() { - type Person struct { - Name string - Age int - Emails []string - Extra map[string]string - } - - // This input can come from anywhere, but typically comes from - // something like decoding JSON where we're not quite sure of the - // struct initially. - input := map[string]interface{}{ - "name": "Mitchell", - "age": 91, - "emails": []string{"one", "two", "three"}, - "extra": map[string]string{ - "twitter": "mitchellh", - }, - } - - var result Person - err := Decode(input, &result) - if err != nil { - panic(err) - } - - fmt.Printf("%#v", result) - // Output: - // mapstructure.Person{Name:"Mitchell", Age:91, Emails:[]string{"one", "two", "three"}, Extra:map[string]string{"twitter":"mitchellh"}} -} - -func ExampleDecode_errors() { - type Person struct { - Name string - Age int - Emails []string - Extra map[string]string - } - - // This input can come from anywhere, but typically comes from - // something like decoding JSON where we're not quite sure of the - // struct initially. - input := map[string]interface{}{ - "name": 123, - "age": "bad value", - "emails": []int{1, 2, 3}, - } - - var result Person - err := Decode(input, &result) - if err == nil { - panic("should have an error") - } - - fmt.Println(err.Error()) - // Output: - // 5 error(s) decoding: - // - // * 'Age' expected type 'int', got unconvertible type 'string' - // * 'Emails[0]' expected type 'string', got unconvertible type 'int' - // * 'Emails[1]' expected type 'string', got unconvertible type 'int' - // * 'Emails[2]' expected type 'string', got unconvertible type 'int' - // * 'Name' expected type 'string', got unconvertible type 'int' -} - -func ExampleDecode_metadata() { - type Person struct { - Name string - Age int - } - - // This input can come from anywhere, but typically comes from - // something like decoding JSON where we're not quite sure of the - // struct initially. - input := map[string]interface{}{ - "name": "Mitchell", - "age": 91, - "email": "foo@bar.com", - } - - // For metadata, we make a more advanced DecoderConfig so we can - // more finely configure the decoder that is used. In this case, we - // just tell the decoder we want to track metadata. - var md Metadata - var result Person - config := &DecoderConfig{ - Metadata: &md, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - panic(err) - } - - if err := decoder.Decode(input); err != nil { - panic(err) - } - - fmt.Printf("Unused keys: %#v", md.Unused) - // Output: - // Unused keys: []string{"email"} -} - -func ExampleDecode_weaklyTypedInput() { - type Person struct { - Name string - Age int - Emails []string - } - - // This input can come from anywhere, but typically comes from - // something like decoding JSON, generated by a weakly typed language - // such as PHP. - input := map[string]interface{}{ - "name": 123, // number => string - "age": "42", // string => number - "emails": map[string]interface{}{}, // empty map => empty array - } - - var result Person - config := &DecoderConfig{ - WeaklyTypedInput: true, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - panic(err) - } - - err = decoder.Decode(input) - if err != nil { - panic(err) - } - - fmt.Printf("%#v", result) - // Output: mapstructure.Person{Name:"123", Age:42, Emails:[]string{}} -} - -func ExampleDecode_tags() { - // Note that the mapstructure tags defined in the struct type - // can indicate which fields the values are mapped to. - type Person struct { - Name string `mapstructure:"person_name"` - Age int `mapstructure:"person_age"` - } - - input := map[string]interface{}{ - "person_name": "Mitchell", - "person_age": 91, - } - - var result Person - err := Decode(input, &result) - if err != nil { - panic(err) - } - - fmt.Printf("%#v", result) - // Output: - // mapstructure.Person{Name:"Mitchell", Age:91} -} - -func ExampleDecode_embeddedStruct() { - // Squashing multiple embedded structs is allowed using the squash tag. - // This is demonstrated by creating a composite struct of multiple types - // and decoding into it. In this case, a person can carry with it both - // a Family and a Location, as well as their own FirstName. - type Family struct { - LastName string - } - type Location struct { - City string - } - type Person struct { - Family `mapstructure:",squash"` - Location `mapstructure:",squash"` - FirstName string - } - - input := map[string]interface{}{ - "FirstName": "Mitchell", - "LastName": "Hashimoto", - "City": "San Francisco", - } - - var result Person - err := Decode(input, &result) - if err != nil { - panic(err) - } - - fmt.Printf("%s %s, %s", result.FirstName, result.LastName, result.City) - // Output: - // Mitchell Hashimoto, San Francisco -} diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure_test.go b/vendor/github.com/mitchellh/mapstructure/mapstructure_test.go deleted file mode 100644 index 45e72849..00000000 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure_test.go +++ /dev/null @@ -1,1047 +0,0 @@ -package mapstructure - -import ( - "reflect" - "sort" - "strings" - "testing" -) - -type Basic struct { - Vstring string - Vint int - Vuint uint - Vbool bool - Vfloat float64 - Vextra string - vsilent bool - Vdata interface{} -} - -type BasicSquash struct { - Test Basic `mapstructure:",squash"` -} - -type Embedded struct { - Basic - Vunique string -} - -type EmbeddedPointer struct { - *Basic - Vunique string -} - -type EmbeddedSquash struct { - Basic `mapstructure:",squash"` - Vunique string -} - -type SquashOnNonStructType struct { - InvalidSquashType int `mapstructure:",squash"` -} - -type Map struct { - Vfoo string - Vother map[string]string -} - -type MapOfStruct struct { - Value map[string]Basic -} - -type Nested struct { - Vfoo string - Vbar Basic -} - -type NestedPointer struct { - Vfoo string - Vbar *Basic -} - -type Slice struct { - Vfoo string - Vbar []string -} - -type SliceOfStruct struct { - Value []Basic -} - -type Tagged struct { - Extra string `mapstructure:"bar,what,what"` - Value string `mapstructure:"foo"` -} - -type TypeConversionResult struct { - IntToFloat float32 - IntToUint uint - IntToBool bool - IntToString string - UintToInt int - UintToFloat float32 - UintToBool bool - UintToString string - BoolToInt int - BoolToUint uint - BoolToFloat float32 - BoolToString string - FloatToInt int - FloatToUint uint - FloatToBool bool - FloatToString string - SliceUint8ToString string - StringToInt int - StringToUint uint - StringToBool bool - StringToFloat float32 - SliceToMap map[string]interface{} - MapToSlice []interface{} -} - -func TestBasicTypes(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": "foo", - "vint": 42, - "Vuint": 42, - "vbool": true, - "Vfloat": 42.42, - "vsilent": true, - "vdata": 42, - } - - var result Basic - err := Decode(input, &result) - if err != nil { - t.Errorf("got an err: %s", err.Error()) - t.FailNow() - } - - if result.Vstring != "foo" { - t.Errorf("vstring value should be 'foo': %#v", result.Vstring) - } - - if result.Vint != 42 { - t.Errorf("vint value should be 42: %#v", result.Vint) - } - - if result.Vuint != 42 { - t.Errorf("vuint value should be 42: %#v", result.Vuint) - } - - if result.Vbool != true { - t.Errorf("vbool value should be true: %#v", result.Vbool) - } - - if result.Vfloat != 42.42 { - t.Errorf("vfloat value should be 42.42: %#v", result.Vfloat) - } - - if result.Vextra != "" { - t.Errorf("vextra value should be empty: %#v", result.Vextra) - } - - if result.vsilent != false { - t.Error("vsilent should not be set, it is unexported") - } - - if result.Vdata != 42 { - t.Error("vdata should be valid") - } -} - -func TestBasic_IntWithFloat(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vint": float64(42), - } - - var result Basic - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err) - } -} - -func TestBasic_Merge(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vint": 42, - } - - var result Basic - result.Vuint = 100 - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err) - } - - expected := Basic{ - Vint: 42, - Vuint: 100, - } - if !reflect.DeepEqual(result, expected) { - t.Fatalf("bad: %#v", result) - } -} - -func TestDecode_BasicSquash(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": "foo", - } - - var result BasicSquash - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err.Error()) - } - - if result.Test.Vstring != "foo" { - t.Errorf("vstring value should be 'foo': %#v", result.Test.Vstring) - } -} - -func TestDecode_Embedded(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": "foo", - "Basic": map[string]interface{}{ - "vstring": "innerfoo", - }, - "vunique": "bar", - } - - var result Embedded - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err.Error()) - } - - if result.Vstring != "innerfoo" { - t.Errorf("vstring value should be 'innerfoo': %#v", result.Vstring) - } - - if result.Vunique != "bar" { - t.Errorf("vunique value should be 'bar': %#v", result.Vunique) - } -} - -func TestDecode_EmbeddedPointer(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": "foo", - "Basic": map[string]interface{}{ - "vstring": "innerfoo", - }, - "vunique": "bar", - } - - var result EmbeddedPointer - err := Decode(input, &result) - if err == nil { - t.Fatal("should get error") - } -} - -func TestDecode_EmbeddedSquash(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": "foo", - "vunique": "bar", - } - - var result EmbeddedSquash - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err.Error()) - } - - if result.Vstring != "foo" { - t.Errorf("vstring value should be 'foo': %#v", result.Vstring) - } - - if result.Vunique != "bar" { - t.Errorf("vunique value should be 'bar': %#v", result.Vunique) - } -} - -func TestDecode_SquashOnNonStructType(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "InvalidSquashType": 42, - } - - var result SquashOnNonStructType - err := Decode(input, &result) - if err == nil { - t.Fatal("unexpected success decoding invalid squash field type") - } else if !strings.Contains(err.Error(), "unsupported type for squash") { - t.Fatalf("unexpected error message for invalid squash field type: %s", err) - } -} - -func TestDecode_DecodeHook(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vint": "WHAT", - } - - decodeHook := func(from reflect.Kind, to reflect.Kind, v interface{}) (interface{}, error) { - if from == reflect.String && to != reflect.String { - return 5, nil - } - - return v, nil - } - - var result Basic - config := &DecoderConfig{ - DecodeHook: decodeHook, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - err = decoder.Decode(input) - if err != nil { - t.Fatalf("got an err: %s", err) - } - - if result.Vint != 5 { - t.Errorf("vint should be 5: %#v", result.Vint) - } -} - -func TestDecode_DecodeHookType(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vint": "WHAT", - } - - decodeHook := func(from reflect.Type, to reflect.Type, v interface{}) (interface{}, error) { - if from.Kind() == reflect.String && - to.Kind() != reflect.String { - return 5, nil - } - - return v, nil - } - - var result Basic - config := &DecoderConfig{ - DecodeHook: decodeHook, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - err = decoder.Decode(input) - if err != nil { - t.Fatalf("got an err: %s", err) - } - - if result.Vint != 5 { - t.Errorf("vint should be 5: %#v", result.Vint) - } -} - -func TestDecode_Nil(t *testing.T) { - t.Parallel() - - var input interface{} = nil - result := Basic{ - Vstring: "foo", - } - - err := Decode(input, &result) - if err != nil { - t.Fatalf("err: %s", err) - } - - if result.Vstring != "foo" { - t.Fatalf("bad: %#v", result.Vstring) - } -} - -func TestDecode_NonStruct(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "foo": "bar", - "bar": "baz", - } - - var result map[string]string - err := Decode(input, &result) - if err != nil { - t.Fatalf("err: %s", err) - } - - if result["foo"] != "bar" { - t.Fatal("foo is not bar") - } -} - -func TestDecode_StructMatch(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vbar": Basic{ - Vstring: "foo", - }, - } - - var result Nested - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err.Error()) - } - - if result.Vbar.Vstring != "foo" { - t.Errorf("bad: %#v", result) - } -} - -func TestDecode_TypeConversion(t *testing.T) { - input := map[string]interface{}{ - "IntToFloat": 42, - "IntToUint": 42, - "IntToBool": 1, - "IntToString": 42, - "UintToInt": 42, - "UintToFloat": 42, - "UintToBool": 42, - "UintToString": 42, - "BoolToInt": true, - "BoolToUint": true, - "BoolToFloat": true, - "BoolToString": true, - "FloatToInt": 42.42, - "FloatToUint": 42.42, - "FloatToBool": 42.42, - "FloatToString": 42.42, - "SliceUint8ToString": []uint8("foo"), - "StringToInt": "42", - "StringToUint": "42", - "StringToBool": "1", - "StringToFloat": "42.42", - "SliceToMap": []interface{}{}, - "MapToSlice": map[string]interface{}{}, - } - - expectedResultStrict := TypeConversionResult{ - IntToFloat: 42.0, - IntToUint: 42, - UintToInt: 42, - UintToFloat: 42, - BoolToInt: 0, - BoolToUint: 0, - BoolToFloat: 0, - FloatToInt: 42, - FloatToUint: 42, - } - - expectedResultWeak := TypeConversionResult{ - IntToFloat: 42.0, - IntToUint: 42, - IntToBool: true, - IntToString: "42", - UintToInt: 42, - UintToFloat: 42, - UintToBool: true, - UintToString: "42", - BoolToInt: 1, - BoolToUint: 1, - BoolToFloat: 1, - BoolToString: "1", - FloatToInt: 42, - FloatToUint: 42, - FloatToBool: true, - FloatToString: "42.42", - SliceUint8ToString: "foo", - StringToInt: 42, - StringToUint: 42, - StringToBool: true, - StringToFloat: 42.42, - SliceToMap: map[string]interface{}{}, - MapToSlice: []interface{}{}, - } - - // Test strict type conversion - var resultStrict TypeConversionResult - err := Decode(input, &resultStrict) - if err == nil { - t.Errorf("should return an error") - } - if !reflect.DeepEqual(resultStrict, expectedResultStrict) { - t.Errorf("expected %v, got: %v", expectedResultStrict, resultStrict) - } - - // Test weak type conversion - var decoder *Decoder - var resultWeak TypeConversionResult - - config := &DecoderConfig{ - WeaklyTypedInput: true, - Result: &resultWeak, - } - - decoder, err = NewDecoder(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - err = decoder.Decode(input) - if err != nil { - t.Fatalf("got an err: %s", err) - } - - if !reflect.DeepEqual(resultWeak, expectedResultWeak) { - t.Errorf("expected \n%#v, got: \n%#v", expectedResultWeak, resultWeak) - } -} - -func TestDecoder_ErrorUnused(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": "hello", - "foo": "bar", - } - - var result Basic - config := &DecoderConfig{ - ErrorUnused: true, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - err = decoder.Decode(input) - if err == nil { - t.Fatal("expected error") - } -} - -func TestMap(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vfoo": "foo", - "vother": map[interface{}]interface{}{ - "foo": "foo", - "bar": "bar", - }, - } - - var result Map - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an error: %s", err) - } - - if result.Vfoo != "foo" { - t.Errorf("vfoo value should be 'foo': %#v", result.Vfoo) - } - - if result.Vother == nil { - t.Fatal("vother should not be nil") - } - - if len(result.Vother) != 2 { - t.Error("vother should have two items") - } - - if result.Vother["foo"] != "foo" { - t.Errorf("'foo' key should be foo, got: %#v", result.Vother["foo"]) - } - - if result.Vother["bar"] != "bar" { - t.Errorf("'bar' key should be bar, got: %#v", result.Vother["bar"]) - } -} - -func TestMapMerge(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vfoo": "foo", - "vother": map[interface{}]interface{}{ - "foo": "foo", - "bar": "bar", - }, - } - - var result Map - result.Vother = map[string]string{"hello": "world"} - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an error: %s", err) - } - - if result.Vfoo != "foo" { - t.Errorf("vfoo value should be 'foo': %#v", result.Vfoo) - } - - expected := map[string]string{ - "foo": "foo", - "bar": "bar", - "hello": "world", - } - if !reflect.DeepEqual(result.Vother, expected) { - t.Errorf("bad: %#v", result.Vother) - } -} - -func TestMapOfStruct(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "value": map[string]interface{}{ - "foo": map[string]string{"vstring": "one"}, - "bar": map[string]string{"vstring": "two"}, - }, - } - - var result MapOfStruct - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err) - } - - if result.Value == nil { - t.Fatal("value should not be nil") - } - - if len(result.Value) != 2 { - t.Error("value should have two items") - } - - if result.Value["foo"].Vstring != "one" { - t.Errorf("foo value should be 'one', got: %s", result.Value["foo"].Vstring) - } - - if result.Value["bar"].Vstring != "two" { - t.Errorf("bar value should be 'two', got: %s", result.Value["bar"].Vstring) - } -} - -func TestNestedType(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vfoo": "foo", - "vbar": map[string]interface{}{ - "vstring": "foo", - "vint": 42, - "vbool": true, - }, - } - - var result Nested - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err.Error()) - } - - if result.Vfoo != "foo" { - t.Errorf("vfoo value should be 'foo': %#v", result.Vfoo) - } - - if result.Vbar.Vstring != "foo" { - t.Errorf("vstring value should be 'foo': %#v", result.Vbar.Vstring) - } - - if result.Vbar.Vint != 42 { - t.Errorf("vint value should be 42: %#v", result.Vbar.Vint) - } - - if result.Vbar.Vbool != true { - t.Errorf("vbool value should be true: %#v", result.Vbar.Vbool) - } - - if result.Vbar.Vextra != "" { - t.Errorf("vextra value should be empty: %#v", result.Vbar.Vextra) - } -} - -func TestNestedTypePointer(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vfoo": "foo", - "vbar": &map[string]interface{}{ - "vstring": "foo", - "vint": 42, - "vbool": true, - }, - } - - var result NestedPointer - err := Decode(input, &result) - if err != nil { - t.Fatalf("got an err: %s", err.Error()) - } - - if result.Vfoo != "foo" { - t.Errorf("vfoo value should be 'foo': %#v", result.Vfoo) - } - - if result.Vbar.Vstring != "foo" { - t.Errorf("vstring value should be 'foo': %#v", result.Vbar.Vstring) - } - - if result.Vbar.Vint != 42 { - t.Errorf("vint value should be 42: %#v", result.Vbar.Vint) - } - - if result.Vbar.Vbool != true { - t.Errorf("vbool value should be true: %#v", result.Vbar.Vbool) - } - - if result.Vbar.Vextra != "" { - t.Errorf("vextra value should be empty: %#v", result.Vbar.Vextra) - } -} - -func TestSlice(t *testing.T) { - t.Parallel() - - inputStringSlice := map[string]interface{}{ - "vfoo": "foo", - "vbar": []string{"foo", "bar", "baz"}, - } - - inputStringSlicePointer := map[string]interface{}{ - "vfoo": "foo", - "vbar": &[]string{"foo", "bar", "baz"}, - } - - outputStringSlice := &Slice{ - "foo", - []string{"foo", "bar", "baz"}, - } - - testSliceInput(t, inputStringSlice, outputStringSlice) - testSliceInput(t, inputStringSlicePointer, outputStringSlice) -} - -func TestInvalidSlice(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vfoo": "foo", - "vbar": 42, - } - - result := Slice{} - err := Decode(input, &result) - if err == nil { - t.Errorf("expected failure") - } -} - -func TestSliceOfStruct(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "value": []map[string]interface{}{ - {"vstring": "one"}, - {"vstring": "two"}, - }, - } - - var result SliceOfStruct - err := Decode(input, &result) - if err != nil { - t.Fatalf("got unexpected error: %s", err) - } - - if len(result.Value) != 2 { - t.Fatalf("expected two values, got %d", len(result.Value)) - } - - if result.Value[0].Vstring != "one" { - t.Errorf("first value should be 'one', got: %s", result.Value[0].Vstring) - } - - if result.Value[1].Vstring != "two" { - t.Errorf("second value should be 'two', got: %s", result.Value[1].Vstring) - } -} - -func TestSliceToMap(t *testing.T) { - t.Parallel() - - input := []map[string]interface{}{ - map[string]interface{}{ - "foo": "bar", - }, - map[string]interface{}{ - "bar": "baz", - }, - } - - var result map[string]interface{} - err := WeakDecode(input, &result) - if err != nil { - t.Fatalf("got an error: %s", err) - } - - expected := map[string]interface{}{ - "foo": "bar", - "bar": "baz", - } - if !reflect.DeepEqual(result, expected) { - t.Errorf("bad: %#v", result) - } -} - -func TestInvalidType(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": 42, - } - - var result Basic - err := Decode(input, &result) - if err == nil { - t.Fatal("error should exist") - } - - derr, ok := err.(*Error) - if !ok { - t.Fatalf("error should be kind of Error, instead: %#v", err) - } - - if derr.Errors[0] != "'Vstring' expected type 'string', got unconvertible type 'int'" { - t.Errorf("got unexpected error: %s", err) - } - - inputNegIntUint := map[string]interface{}{ - "vuint": -42, - } - - err = Decode(inputNegIntUint, &result) - if err == nil { - t.Fatal("error should exist") - } - - derr, ok = err.(*Error) - if !ok { - t.Fatalf("error should be kind of Error, instead: %#v", err) - } - - if derr.Errors[0] != "cannot parse 'Vuint', -42 overflows uint" { - t.Errorf("got unexpected error: %s", err) - } - - inputNegFloatUint := map[string]interface{}{ - "vuint": -42.0, - } - - err = Decode(inputNegFloatUint, &result) - if err == nil { - t.Fatal("error should exist") - } - - derr, ok = err.(*Error) - if !ok { - t.Fatalf("error should be kind of Error, instead: %#v", err) - } - - if derr.Errors[0] != "cannot parse 'Vuint', -42.000000 overflows uint" { - t.Errorf("got unexpected error: %s", err) - } -} - -func TestMetadata(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vfoo": "foo", - "vbar": map[string]interface{}{ - "vstring": "foo", - "Vuint": 42, - "foo": "bar", - }, - "bar": "nil", - } - - var md Metadata - var result Nested - config := &DecoderConfig{ - Metadata: &md, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - err = decoder.Decode(input) - if err != nil { - t.Fatalf("err: %s", err.Error()) - } - - expectedKeys := []string{"Vbar", "Vbar.Vstring", "Vbar.Vuint", "Vfoo"} - sort.Strings(md.Keys) - if !reflect.DeepEqual(md.Keys, expectedKeys) { - t.Fatalf("bad keys: %#v", md.Keys) - } - - expectedUnused := []string{"Vbar.foo", "bar"} - if !reflect.DeepEqual(md.Unused, expectedUnused) { - t.Fatalf("bad unused: %#v", md.Unused) - } -} - -func TestMetadata_Embedded(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "vstring": "foo", - "vunique": "bar", - } - - var md Metadata - var result EmbeddedSquash - config := &DecoderConfig{ - Metadata: &md, - Result: &result, - } - - decoder, err := NewDecoder(config) - if err != nil { - t.Fatalf("err: %s", err) - } - - err = decoder.Decode(input) - if err != nil { - t.Fatalf("err: %s", err.Error()) - } - - expectedKeys := []string{"Vstring", "Vunique"} - - sort.Strings(md.Keys) - if !reflect.DeepEqual(md.Keys, expectedKeys) { - t.Fatalf("bad keys: %#v", md.Keys) - } - - expectedUnused := []string{} - if !reflect.DeepEqual(md.Unused, expectedUnused) { - t.Fatalf("bad unused: %#v", md.Unused) - } -} - -func TestNonPtrValue(t *testing.T) { - t.Parallel() - - err := Decode(map[string]interface{}{}, Basic{}) - if err == nil { - t.Fatal("error should exist") - } - - if err.Error() != "result must be a pointer" { - t.Errorf("got unexpected error: %s", err) - } -} - -func TestTagged(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "foo": "bar", - "bar": "value", - } - - var result Tagged - err := Decode(input, &result) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } - - if result.Value != "bar" { - t.Errorf("value should be 'bar', got: %#v", result.Value) - } - - if result.Extra != "value" { - t.Errorf("extra should be 'value', got: %#v", result.Extra) - } -} - -func TestWeakDecode(t *testing.T) { - t.Parallel() - - input := map[string]interface{}{ - "foo": "4", - "bar": "value", - } - - var result struct { - Foo int - Bar string - } - - if err := WeakDecode(input, &result); err != nil { - t.Fatalf("err: %s", err) - } - if result.Foo != 4 { - t.Fatalf("bad: %#v", result) - } - if result.Bar != "value" { - t.Fatalf("bad: %#v", result) - } -} - -func testSliceInput(t *testing.T, input map[string]interface{}, expected *Slice) { - var result Slice - err := Decode(input, &result) - if err != nil { - t.Fatalf("got error: %s", err) - } - - if result.Vfoo != expected.Vfoo { - t.Errorf("Vfoo expected '%s', got '%s'", expected.Vfoo, result.Vfoo) - } - - if result.Vbar == nil { - t.Fatalf("Vbar a slice, got '%#v'", result.Vbar) - } - - if len(result.Vbar) != len(expected.Vbar) { - t.Errorf("Vbar length should be %d, got %d", len(expected.Vbar), len(result.Vbar)) - } - - for i, v := range result.Vbar { - if v != expected.Vbar[i] { - t.Errorf( - "Vbar[%d] should be '%#v', got '%#v'", - i, expected.Vbar[i], v) - } - } -} diff --git a/vendor/github.com/pelletier/go-buffruneio/README.md b/vendor/github.com/pelletier/go-buffruneio/README.md new file mode 100644 index 00000000..ff608b3a --- /dev/null +++ b/vendor/github.com/pelletier/go-buffruneio/README.md @@ -0,0 +1,62 @@ +# buffruneio + +[![Tests Status](https://travis-ci.org/pelletier/go-buffruneio.svg?branch=master)](https://travis-ci.org/pelletier/go-buffruneio) +[![GoDoc](https://godoc.org/github.com/pelletier/go-buffruneio?status.svg)](https://godoc.org/github.com/pelletier/go-buffruneio) + +Buffruneio is a wrapper around bufio to provide buffered runes access with +unlimited unreads. + +```go +import "github.com/pelletier/go-buffruneio" +``` + +## Examples + +```go +import ( + "fmt" + "github.com/pelletier/go-buffruneio" + "strings" +) + +reader := buffruneio.NewReader(strings.NewReader("abcd")) +fmt.Println(reader.ReadRune()) // 'a' +fmt.Println(reader.ReadRune()) // 'b' +fmt.Println(reader.ReadRune()) // 'c' +reader.UnreadRune() +reader.UnreadRune() +fmt.Println(reader.ReadRune()) // 'b' +fmt.Println(reader.ReadRune()) // 'c' +``` + +## Documentation + +The documentation and additional examples are available at +[godoc.org](http://godoc.org/github.com/pelletier/go-buffruneio). + +## Contribute + +Feel free to report bugs and patches using GitHub's pull requests system on +[pelletier/go-toml](https://github.com/pelletier/go-buffruneio). Any feedback is +much appreciated! + +## LICENSE + +Copyright (c) 2016 Thomas Pelletier + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/pelletier/go-buffruneio/buffruneio.go b/vendor/github.com/pelletier/go-buffruneio/buffruneio.go new file mode 100644 index 00000000..4e6d6ea6 --- /dev/null +++ b/vendor/github.com/pelletier/go-buffruneio/buffruneio.go @@ -0,0 +1,117 @@ +// Package buffruneio is a wrapper around bufio to provide buffered runes access with unlimited unreads. +package buffruneio + +import ( + "bufio" + "container/list" + "errors" + "io" +) + +// Rune to indicate end of file. +const ( + EOF = -(iota + 1) +) + +// ErrNoRuneToUnread is returned by UnreadRune() when the read index is already at the beginning of the buffer. +var ErrNoRuneToUnread = errors.New("no rune to unwind") + +// Reader implements runes buffering for an io.Reader object. +type Reader struct { + buffer *list.List + current *list.Element + input *bufio.Reader +} + +// NewReader returns a new Reader. +func NewReader(rd io.Reader) *Reader { + return &Reader{ + buffer: list.New(), + input: bufio.NewReader(rd), + } +} + +type runeWithSize struct { + r rune + size int +} + +func (rd *Reader) feedBuffer() error { + r, size, err := rd.input.ReadRune() + + if err != nil { + if err != io.EOF { + return err + } + r = EOF + } + + newRuneWithSize := runeWithSize{r, size} + + rd.buffer.PushBack(newRuneWithSize) + if rd.current == nil { + rd.current = rd.buffer.Back() + } + return nil +} + +// ReadRune reads the next rune from buffer, or from the underlying reader if needed. +func (rd *Reader) ReadRune() (rune, int, error) { + if rd.current == rd.buffer.Back() || rd.current == nil { + err := rd.feedBuffer() + if err != nil { + return EOF, 0, err + } + } + + runeWithSize := rd.current.Value.(runeWithSize) + rd.current = rd.current.Next() + return runeWithSize.r, runeWithSize.size, nil +} + +// UnreadRune pushes back the previously read rune in the buffer, extending it if needed. +func (rd *Reader) UnreadRune() error { + if rd.current == rd.buffer.Front() { + return ErrNoRuneToUnread + } + if rd.current == nil { + rd.current = rd.buffer.Back() + } else { + rd.current = rd.current.Prev() + } + return nil +} + +// Forget removes runes stored before the current stream position index. +func (rd *Reader) Forget() { + if rd.current == nil { + rd.current = rd.buffer.Back() + } + for ; rd.current != rd.buffer.Front(); rd.buffer.Remove(rd.current.Prev()) { + } +} + +// PeekRune returns at most the next n runes, reading from the uderlying source if +// needed. Does not move the current index. It includes EOF if reached. +func (rd *Reader) PeekRunes(n int) []rune { + res := make([]rune, 0, n) + cursor := rd.current + for i := 0; i < n; i++ { + if cursor == nil { + err := rd.feedBuffer() + if err != nil { + return res + } + cursor = rd.buffer.Back() + } + if cursor != nil { + r := cursor.Value.(runeWithSize).r + res = append(res, r) + if r == EOF { + return res + } + cursor = cursor.Next() + } + } + return res +} diff --git a/vendor/github.com/pelletier/go-toml/LICENSE b/vendor/github.com/pelletier/go-toml/LICENSE new file mode 100644 index 00000000..583bdae6 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/pelletier/go-toml/README.md b/vendor/github.com/pelletier/go-toml/README.md new file mode 100644 index 00000000..b8137e02 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/README.md @@ -0,0 +1,120 @@ +# go-toml + +Go library for the [TOML](https://github.com/mojombo/toml) format. + +This library supports TOML version +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) + +[![GoDoc](https://godoc.org/github.com/pelletier/go-toml?status.svg)](http://godoc.org/github.com/pelletier/go-toml) +[![license](https://img.shields.io/github/license/pelletier/go-toml.svg)](https://github.com/pelletier/go-toml/blob/master/LICENSE) +[![Build Status](https://travis-ci.org/pelletier/go-toml.svg?branch=master)](https://travis-ci.org/pelletier/go-toml) +[![Coverage Status](https://coveralls.io/repos/github/pelletier/go-toml/badge.svg?branch=master)](https://coveralls.io/github/pelletier/go-toml?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/pelletier/go-toml)](https://goreportcard.com/report/github.com/pelletier/go-toml) + +## Features + +Go-toml provides the following features for using data parsed from TOML documents: + +* Load TOML documents from files and string data +* Easily navigate TOML structure using TomlTree +* Line & column position data for all parsed elements +* Query support similar to JSON-Path +* Syntax errors contain line and column numbers + +Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing: + +* Semantic evaluation of parsed TOML +* Informing a user of mistakes in the source document, after it has been parsed +* Programatic handling of default values on a case-by-case basis +* Using a TOML document as a flexible data-store + +## Import + + import "github.com/pelletier/go-toml" + +## Usage + +### Example + +Say you have a TOML file that looks like this: + +```toml +[postgres] +user = "pelletier" +password = "mypassword" +``` + +Read the username and password like this: + +```go +import ( + "fmt" + "github.com/pelletier/go-toml" +) + +config, err := toml.LoadFile("config.toml") +if err != nil { + fmt.Println("Error ", err.Error()) +} else { + // retrieve data directly + user := config.Get("postgres.user").(string) + password := config.Get("postgres.password").(string) + + // or using an intermediate object + configTree := config.Get("postgres").(*toml.TomlTree) + user = configTree.Get("user").(string) + password = configTree.Get("password").(string) + fmt.Println("User is ", user, ". Password is ", password) + + // show where elements are in the file + fmt.Println("User position: %v", configTree.GetPosition("user")) + fmt.Println("Password position: %v", configTree.GetPosition("password")) + + // use a query to gather elements without walking the tree + results, _ := config.Query("$..[user,password]") + for ii, item := range results.Values() { + fmt.Println("Query result %d: %v", ii, item) + } +} +``` + +## Documentation + +The documentation and additional examples are available at +[godoc.org](http://godoc.org/github.com/pelletier/go-toml). + +## Tools + +Go-toml provides two handy command line tools: + +* `tomll`: Reads TOML files and lint them. + + ``` + go install github.com/pelletier/go-toml/cmd/tomll + tomll --help + ``` +* `tomljson`: Reads a TOML file and outputs its JSON representation. + + ``` + go install github.com/pelletier/go-toml/cmd/tomljson + tomljson --help + ``` + +## Contribute + +Feel free to report bugs and patches using GitHub's pull requests system on +[pelletier/go-toml](https://github.com/pelletier/go-toml). Any feedback would be +much appreciated! + +### Run tests + +You have to make sure two kind of tests run: + +1. The Go unit tests +2. The TOML examples base + +You can run both of them using `./test.sh`. + +## License + +The MIT License (MIT). Read [LICENSE](LICENSE). diff --git a/vendor/github.com/pelletier/go-toml/clean.sh b/vendor/github.com/pelletier/go-toml/clean.sh new file mode 100755 index 00000000..44d49d93 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/clean.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# fail out of the script if anything here fails +set -e + +# clear out stuff generated by test.sh +rm -rf src test_program_bin toml-test diff --git a/vendor/github.com/pelletier/go-toml/doc.go b/vendor/github.com/pelletier/go-toml/doc.go new file mode 100644 index 00000000..9156b736 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/doc.go @@ -0,0 +1,250 @@ +// Package toml is a TOML markup language parser. +// +// This version supports the specification as described in +// https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md +// +// TOML Parsing +// +// TOML data may be parsed in two ways: by file, or by string. +// +// // load TOML data by filename +// tree, err := toml.LoadFile("filename.toml") +// +// // load TOML data stored in a string +// tree, err := toml.Load(stringContainingTomlData) +// +// Either way, the result is a TomlTree object that can be used to navigate the +// structure and data within the original document. +// +// +// Getting data from the TomlTree +// +// After parsing TOML data with Load() or LoadFile(), use the Has() and Get() +// methods on the returned TomlTree, to find your way through the document data. +// +// if tree.Has("foo") { +// fmt.Println("foo is:", tree.Get("foo")) +// } +// +// Working with Paths +// +// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set() +// and GetDefault() methods. These are the same kind of key paths used within the +// TOML specification for struct tames. +// +// // looks for a key named 'baz', within struct 'bar', within struct 'foo' +// tree.Has("foo.bar.baz") +// +// // returns the key at this path, if it is there +// tree.Get("foo.bar.baz") +// +// TOML allows keys to contain '.', which can cause this syntax to be problematic +// for some documents. In such cases, use the GetPath(), HasPath(), and SetPath(), +// methods to explicitly define the path. This form is also faster, since +// it avoids having to parse the passed key for '.' delimiters. +// +// // looks for a key named 'baz', within struct 'bar', within struct 'foo' +// tree.HasPath([]string{"foo","bar","baz"}) +// +// // returns the key at this path, if it is there +// tree.GetPath([]string{"foo","bar","baz"}) +// +// Note that this is distinct from the heavyweight query syntax supported by +// TomlTree.Query() and the Query() struct (see below). +// +// Position Support +// +// Each element within the TomlTree is stored with position metadata, which is +// invaluable for providing semantic feedback to a user. This helps in +// situations where the TOML file parses correctly, but contains data that is +// not correct for the application. In such cases, an error message can be +// generated that indicates the problem line and column number in the source +// TOML document. +// +// // load TOML data +// tree, _ := toml.Load("filename.toml") +// +// // get an entry and report an error if it's the wrong type +// element := tree.Get("foo") +// if value, ok := element.(int64); !ok { +// return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo")) +// } +// +// // report an error if an expected element is missing +// if !tree.Has("bar") { +// return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition("")) +// } +// +// Query Support +// +// The TOML query path implementation is based loosely on the JSONPath specification: +// http://goessner.net/articles/JsonPath/ +// +// The idea behind a query path is to allow quick access to any element, or set +// of elements within TOML document, with a single expression. +// +// result, err := tree.Query("$.foo.bar.baz") +// +// This is roughly equivalent to: +// +// next := tree.Get("foo") +// if next != nil { +// next = next.Get("bar") +// if next != nil { +// next = next.Get("baz") +// } +// } +// result := next +// +// err is nil if any parsing exception occurs. +// +// If no node in the tree matches the query, result will simply contain an empty list of +// items. +// +// As illustrated above, the query path is much more efficient, especially since +// the structure of the TOML file can vary. Rather than making assumptions about +// a document's structure, a query allows the programmer to make structured +// requests into the document, and get zero or more values as a result. +// +// The syntax of a query begins with a root token, followed by any number +// sub-expressions: +// +// $ +// Root of the TOML tree. This must always come first. +// .name +// Selects child of this node, where 'name' is a TOML key +// name. +// ['name'] +// Selects child of this node, where 'name' is a string +// containing a TOML key name. +// [index] +// Selcts child array element at 'index'. +// ..expr +// Recursively selects all children, filtered by an a union, +// index, or slice expression. +// ..* +// Recursive selection of all nodes at this point in the +// tree. +// .* +// Selects all children of the current node. +// [expr,expr] +// Union operator - a logical 'or' grouping of two or more +// sub-expressions: index, key name, or filter. +// [start:end:step] +// Slice operator - selects array elements from start to +// end-1, at the given step. All three arguments are +// optional. +// [?(filter)] +// Named filter expression - the function 'filter' is +// used to filter children at this node. +// +// Query Indexes And Slices +// +// Index expressions perform no bounds checking, and will contribute no +// values to the result set if the provided index or index range is invalid. +// Negative indexes represent values from the end of the array, counting backwards. +// +// // select the last index of the array named 'foo' +// tree.Query("$.foo[-1]") +// +// Slice expressions are supported, by using ':' to separate a start/end index pair. +// +// // select up to the first five elements in the array +// tree.Query("$.foo[0:5]") +// +// Slice expressions also allow negative indexes for the start and stop +// arguments. +// +// // select all array elements. +// tree.Query("$.foo[0:-1]") +// +// Slice expressions may have an optional stride/step parameter: +// +// // select every other element +// tree.Query("$.foo[0:-1:2]") +// +// Slice start and end parameters are also optional: +// +// // these are all equivalent and select all the values in the array +// tree.Query("$.foo[:]") +// tree.Query("$.foo[0:]") +// tree.Query("$.foo[:-1]") +// tree.Query("$.foo[0:-1:]") +// tree.Query("$.foo[::1]") +// tree.Query("$.foo[0::1]") +// tree.Query("$.foo[:-1:1]") +// tree.Query("$.foo[0:-1:1]") +// +// Query Filters +// +// Query filters are used within a Union [,] or single Filter [] expression. +// A filter only allows nodes that qualify through to the next expression, +// and/or into the result set. +// +// // returns children of foo that are permitted by the 'bar' filter. +// tree.Query("$.foo[?(bar)]") +// +// There are several filters provided with the library: +// +// tree +// Allows nodes of type TomlTree. +// int +// Allows nodes of type int64. +// float +// Allows nodes of type float64. +// string +// Allows nodes of type string. +// time +// Allows nodes of type time.Time. +// bool +// Allows nodes of type bool. +// +// Query Results +// +// An executed query returns a QueryResult object. This contains the nodes +// in the TOML tree that qualify the query expression. Position information +// is also available for each value in the set. +// +// // display the results of a query +// results := tree.Query("$.foo.bar.baz") +// for idx, value := results.Values() { +// fmt.Println("%v: %v", results.Positions()[idx], value) +// } +// +// Compiled Queries +// +// Queries may be executed directly on a TomlTree object, or compiled ahead +// of time and executed discretely. The former is more convienent, but has the +// penalty of having to recompile the query expression each time. +// +// // basic query +// results := tree.Query("$.foo.bar.baz") +// +// // compiled query +// query := toml.CompileQuery("$.foo.bar.baz") +// results := query.Execute(tree) +// +// // run the compiled query again on a different tree +// moreResults := query.Execute(anotherTree) +// +// User Defined Query Filters +// +// Filter expressions may also be user defined by using the SetFilter() +// function on the Query object. The function must return true/false, which +// signifies if the passed node is kept or discarded, respectively. +// +// // create a query that references a user-defined filter +// query, _ := CompileQuery("$[?(bazOnly)]") +// +// // define the filter, and assign it to the query +// query.SetFilter("bazOnly", func(node interface{}) bool{ +// if tree, ok := node.(*TomlTree); ok { +// return tree.Has("baz") +// } +// return false // reject all other node types +// }) +// +// // run the query +// query.Execute(tree) +// +package toml diff --git a/vendor/github.com/BurntSushi/toml/_examples/invalid.toml b/vendor/github.com/pelletier/go-toml/example-crlf.toml similarity index 71% rename from vendor/github.com/BurntSushi/toml/_examples/invalid.toml rename to vendor/github.com/pelletier/go-toml/example-crlf.toml index beb1dba5..12950a16 100644 --- a/vendor/github.com/BurntSushi/toml/_examples/invalid.toml +++ b/vendor/github.com/pelletier/go-toml/example-crlf.toml @@ -1,35 +1,29 @@ -# This is an INVALID TOML document. Boom. -# Can you spot the error without help? - -title = "TOML Example" - -[owner] -name = "Tom Preston-Werner" -organization = "GitHub" -bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." -dob = 1979-05-27T7:32:00Z # First class dates? Why not? - -[database] -server = "192.168.1.1" -ports = [ 8001, 8001, 8002 ] -connection_max = 5000 -enabled = true - -[servers] - # You can indent as you please. Tabs or spaces. TOML don't care. - [servers.alpha] - ip = "10.0.0.1" - dc = "eqdc10" - - [servers.beta] - ip = "10.0.0.2" - dc = "eqdc10" - -[clients] -data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it - -# Line breaks are OK when inside arrays -hosts = [ - "alpha", - "omega" -] +# This is a TOML document. Boom. + +title = "TOML Example" + +[owner] +name = "Tom Preston-Werner" +organization = "GitHub" +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? + +[database] +server = "192.168.1.1" +ports = [ 8001, 8001, 8002 ] +connection_max = 5000 +enabled = true + +[servers] + + # You can indent as you please. Tabs or spaces. TOML don't care. + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + +[clients] +data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it diff --git a/vendor/github.com/BurntSushi/toml/_examples/example.toml b/vendor/github.com/pelletier/go-toml/example.toml similarity index 89% rename from vendor/github.com/BurntSushi/toml/_examples/example.toml rename to vendor/github.com/pelletier/go-toml/example.toml index 32c7a4fa..3d902f28 100644 --- a/vendor/github.com/BurntSushi/toml/_examples/example.toml +++ b/vendor/github.com/pelletier/go-toml/example.toml @@ -27,9 +27,3 @@ enabled = true [clients] data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it - -# Line breaks are OK when inside arrays -hosts = [ - "alpha", - "omega" -] diff --git a/vendor/github.com/pelletier/go-toml/keysparsing.go b/vendor/github.com/pelletier/go-toml/keysparsing.go new file mode 100644 index 00000000..d62ca5fd --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/keysparsing.go @@ -0,0 +1,94 @@ +// Parsing keys handling both bare and quoted keys. + +package toml + +import ( + "bytes" + "errors" + "fmt" + "unicode" +) + +func parseKey(key string) ([]string, error) { + groups := []string{} + var buffer bytes.Buffer + inQuotes := false + wasInQuotes := false + escapeNext := false + ignoreSpace := true + expectDot := false + + for _, char := range key { + if ignoreSpace { + if char == ' ' { + continue + } + ignoreSpace = false + } + if escapeNext { + buffer.WriteRune(char) + escapeNext = false + continue + } + switch char { + case '\\': + escapeNext = true + continue + case '"': + if inQuotes { + groups = append(groups, buffer.String()) + buffer.Reset() + wasInQuotes = true + } + inQuotes = !inQuotes + expectDot = false + case '.': + if inQuotes { + buffer.WriteRune(char) + } else { + if !wasInQuotes { + if buffer.Len() == 0 { + return nil, errors.New("empty table key") + } + groups = append(groups, buffer.String()) + buffer.Reset() + } + ignoreSpace = true + expectDot = false + wasInQuotes = false + } + case ' ': + if inQuotes { + buffer.WriteRune(char) + } else { + expectDot = true + } + default: + if !inQuotes && !isValidBareChar(char) { + return nil, fmt.Errorf("invalid bare character: %c", char) + } + if !inQuotes && expectDot { + return nil, errors.New("what?") + } + buffer.WriteRune(char) + expectDot = false + } + } + if inQuotes { + return nil, errors.New("mismatched quotes") + } + if escapeNext { + return nil, errors.New("unfinished escape sequence") + } + if buffer.Len() > 0 { + groups = append(groups, buffer.String()) + } + if len(groups) == 0 { + return nil, errors.New("empty key") + } + return groups, nil +} + +func isValidBareChar(r rune) bool { + return isAlphanumeric(r) || r == '-' || unicode.IsNumber(r) +} diff --git a/vendor/github.com/pelletier/go-toml/lexer.go b/vendor/github.com/pelletier/go-toml/lexer.go new file mode 100644 index 00000000..104f3b1f --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/lexer.go @@ -0,0 +1,657 @@ +// TOML lexer. +// +// Written using the principles developed by Rob Pike in +// http://www.youtube.com/watch?v=HxaD_trXwRE + +package toml + +import ( + "errors" + "fmt" + "io" + "regexp" + "strconv" + "strings" + + "github.com/pelletier/go-buffruneio" +) + +var dateRegexp *regexp.Regexp + +// Define state functions +type tomlLexStateFn func() tomlLexStateFn + +// Define lexer +type tomlLexer struct { + input *buffruneio.Reader // Textual source + buffer []rune // Runes composing the current token + tokens chan token + depth int + line int + col int + endbufferLine int + endbufferCol int +} + +// Basic read operations on input + +func (l *tomlLexer) read() rune { + r, _, err := l.input.ReadRune() + if err != nil { + panic(err) + } + if r == '\n' { + l.endbufferLine++ + l.endbufferCol = 1 + } else { + l.endbufferCol++ + } + return r +} + +func (l *tomlLexer) next() rune { + r := l.read() + + if r != eof { + l.buffer = append(l.buffer, r) + } + return r +} + +func (l *tomlLexer) ignore() { + l.buffer = make([]rune, 0) + l.line = l.endbufferLine + l.col = l.endbufferCol +} + +func (l *tomlLexer) skip() { + l.next() + l.ignore() +} + +func (l *tomlLexer) fastForward(n int) { + for i := 0; i < n; i++ { + l.next() + } +} + +func (l *tomlLexer) emitWithValue(t tokenType, value string) { + l.tokens <- token{ + Position: Position{l.line, l.col}, + typ: t, + val: value, + } + l.ignore() +} + +func (l *tomlLexer) emit(t tokenType) { + l.emitWithValue(t, string(l.buffer)) +} + +func (l *tomlLexer) peek() rune { + r, _, err := l.input.ReadRune() + if err != nil { + panic(err) + } + l.input.UnreadRune() + return r +} + +func (l *tomlLexer) follow(next string) bool { + for _, expectedRune := range next { + r, _, err := l.input.ReadRune() + defer l.input.UnreadRune() + if err != nil { + panic(err) + } + if expectedRune != r { + return false + } + } + return true +} + +// Error management + +func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn { + l.tokens <- token{ + Position: Position{l.line, l.col}, + typ: tokenError, + val: fmt.Sprintf(format, args...), + } + return nil +} + +// State functions + +func (l *tomlLexer) lexVoid() tomlLexStateFn { + for { + next := l.peek() + switch next { + case '[': + return l.lexTableKey + case '#': + return l.lexComment(l.lexVoid) + case '=': + return l.lexEqual + case '\r': + fallthrough + case '\n': + l.skip() + continue + } + + if isSpace(next) { + l.skip() + } + + if l.depth > 0 { + return l.lexRvalue + } + + if isKeyStartChar(next) { + return l.lexKey + } + + if next == eof { + l.next() + break + } + } + + l.emit(tokenEOF) + return nil +} + +func (l *tomlLexer) lexRvalue() tomlLexStateFn { + for { + next := l.peek() + switch next { + case '.': + return l.errorf("cannot start float with a dot") + case '=': + return l.lexEqual + case '[': + l.depth++ + return l.lexLeftBracket + case ']': + l.depth-- + return l.lexRightBracket + case '{': + return l.lexLeftCurlyBrace + case '}': + return l.lexRightCurlyBrace + case '#': + return l.lexComment(l.lexRvalue) + case '"': + return l.lexString + case '\'': + return l.lexLiteralString + case ',': + return l.lexComma + case '\r': + fallthrough + case '\n': + l.skip() + if l.depth == 0 { + return l.lexVoid + } + return l.lexRvalue + case '_': + return l.errorf("cannot start number with underscore") + } + + if l.follow("true") { + return l.lexTrue + } + + if l.follow("false") { + return l.lexFalse + } + + if isSpace(next) { + l.skip() + continue + } + + if next == eof { + l.next() + break + } + + possibleDate := string(l.input.PeekRunes(35)) + dateMatch := dateRegexp.FindString(possibleDate) + if dateMatch != "" { + l.fastForward(len(dateMatch)) + return l.lexDate + } + + if next == '+' || next == '-' || isDigit(next) { + return l.lexNumber + } + + if isAlphanumeric(next) { + return l.lexKey + } + + return l.errorf("no value can start with %c", next) + } + + l.emit(tokenEOF) + return nil +} + +func (l *tomlLexer) lexLeftCurlyBrace() tomlLexStateFn { + l.next() + l.emit(tokenLeftCurlyBrace) + return l.lexRvalue +} + +func (l *tomlLexer) lexRightCurlyBrace() tomlLexStateFn { + l.next() + l.emit(tokenRightCurlyBrace) + return l.lexRvalue +} + +func (l *tomlLexer) lexDate() tomlLexStateFn { + l.emit(tokenDate) + return l.lexRvalue +} + +func (l *tomlLexer) lexTrue() tomlLexStateFn { + l.fastForward(4) + l.emit(tokenTrue) + return l.lexRvalue +} + +func (l *tomlLexer) lexFalse() tomlLexStateFn { + l.fastForward(5) + l.emit(tokenFalse) + return l.lexRvalue +} + +func (l *tomlLexer) lexEqual() tomlLexStateFn { + l.next() + l.emit(tokenEqual) + return l.lexRvalue +} + +func (l *tomlLexer) lexComma() tomlLexStateFn { + l.next() + l.emit(tokenComma) + return l.lexRvalue +} + +func (l *tomlLexer) lexKey() tomlLexStateFn { + growingString := "" + + for r := l.peek(); isKeyChar(r) || r == '\n' || r == '\r'; r = l.peek() { + if r == '"' { + l.next() + str, err := l.lexStringAsString(`"`, false, true) + if err != nil { + return l.errorf(err.Error()) + } + growingString += `"` + str + `"` + l.next() + continue + } else if r == '\n' { + return l.errorf("keys cannot contain new lines") + } else if isSpace(r) { + break + } else if !isValidBareChar(r) { + return l.errorf("keys cannot contain %c character", r) + } + growingString += string(r) + l.next() + } + l.emitWithValue(tokenKey, growingString) + return l.lexVoid +} + +func (l *tomlLexer) lexComment(previousState tomlLexStateFn) tomlLexStateFn { + return func() tomlLexStateFn { + for next := l.peek(); next != '\n' && next != eof; next = l.peek() { + if next == '\r' && l.follow("\r\n") { + break + } + l.next() + } + l.ignore() + return previousState + } +} + +func (l *tomlLexer) lexLeftBracket() tomlLexStateFn { + l.next() + l.emit(tokenLeftBracket) + return l.lexRvalue +} + +func (l *tomlLexer) lexLiteralStringAsString(terminator string, discardLeadingNewLine bool) (string, error) { + growingString := "" + + if discardLeadingNewLine { + if l.follow("\r\n") { + l.skip() + l.skip() + } else if l.peek() == '\n' { + l.skip() + } + } + + // find end of string + for { + if l.follow(terminator) { + return growingString, nil + } + + next := l.peek() + if next == eof { + break + } + growingString += string(l.next()) + } + + return "", errors.New("unclosed string") +} + +func (l *tomlLexer) lexLiteralString() tomlLexStateFn { + l.skip() + + // handle special case for triple-quote + terminator := "'" + discardLeadingNewLine := false + if l.follow("''") { + l.skip() + l.skip() + terminator = "'''" + discardLeadingNewLine = true + } + + str, err := l.lexLiteralStringAsString(terminator, discardLeadingNewLine) + if err != nil { + return l.errorf(err.Error()) + } + + l.emitWithValue(tokenString, str) + l.fastForward(len(terminator)) + l.ignore() + return l.lexRvalue +} + +// Lex a string and return the results as a string. +// Terminator is the substring indicating the end of the token. +// The resulting string does not include the terminator. +func (l *tomlLexer) lexStringAsString(terminator string, discardLeadingNewLine, acceptNewLines bool) (string, error) { + growingString := "" + + if discardLeadingNewLine { + if l.follow("\r\n") { + l.skip() + l.skip() + } else if l.peek() == '\n' { + l.skip() + } + } + + for { + if l.follow(terminator) { + return growingString, nil + } + + if l.follow("\\") { + l.next() + switch l.peek() { + case '\r': + fallthrough + case '\n': + fallthrough + case '\t': + fallthrough + case ' ': + // skip all whitespace chars following backslash + for strings.ContainsRune("\r\n\t ", l.peek()) { + l.next() + } + case '"': + growingString += "\"" + l.next() + case 'n': + growingString += "\n" + l.next() + case 'b': + growingString += "\b" + l.next() + case 'f': + growingString += "\f" + l.next() + case '/': + growingString += "/" + l.next() + case 't': + growingString += "\t" + l.next() + case 'r': + growingString += "\r" + l.next() + case '\\': + growingString += "\\" + l.next() + case 'u': + l.next() + code := "" + for i := 0; i < 4; i++ { + c := l.peek() + if !isHexDigit(c) { + return "", errors.New("unfinished unicode escape") + } + l.next() + code = code + string(c) + } + intcode, err := strconv.ParseInt(code, 16, 32) + if err != nil { + return "", errors.New("invalid unicode escape: \\u" + code) + } + growingString += string(rune(intcode)) + case 'U': + l.next() + code := "" + for i := 0; i < 8; i++ { + c := l.peek() + if !isHexDigit(c) { + return "", errors.New("unfinished unicode escape") + } + l.next() + code = code + string(c) + } + intcode, err := strconv.ParseInt(code, 16, 64) + if err != nil { + return "", errors.New("invalid unicode escape: \\U" + code) + } + growingString += string(rune(intcode)) + default: + return "", errors.New("invalid escape sequence: \\" + string(l.peek())) + } + } else { + r := l.peek() + + if 0x00 <= r && r <= 0x1F && !(acceptNewLines && (r == '\n' || r == '\r')) { + return "", fmt.Errorf("unescaped control character %U", r) + } + l.next() + growingString += string(r) + } + + if l.peek() == eof { + break + } + } + + return "", errors.New("unclosed string") +} + +func (l *tomlLexer) lexString() tomlLexStateFn { + l.skip() + + // handle special case for triple-quote + terminator := `"` + discardLeadingNewLine := false + acceptNewLines := false + if l.follow(`""`) { + l.skip() + l.skip() + terminator = `"""` + discardLeadingNewLine = true + acceptNewLines = true + } + + str, err := l.lexStringAsString(terminator, discardLeadingNewLine, acceptNewLines) + + if err != nil { + return l.errorf(err.Error()) + } + + l.emitWithValue(tokenString, str) + l.fastForward(len(terminator)) + l.ignore() + return l.lexRvalue +} + +func (l *tomlLexer) lexTableKey() tomlLexStateFn { + l.next() + + if l.peek() == '[' { + // token '[[' signifies an array of tables + l.next() + l.emit(tokenDoubleLeftBracket) + return l.lexInsideTableArrayKey + } + // vanilla table key + l.emit(tokenLeftBracket) + return l.lexInsideTableKey +} + +func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn { + for r := l.peek(); r != eof; r = l.peek() { + switch r { + case ']': + if len(l.buffer) > 0 { + l.emit(tokenKeyGroupArray) + } + l.next() + if l.peek() != ']' { + break + } + l.next() + l.emit(tokenDoubleRightBracket) + return l.lexVoid + case '[': + return l.errorf("table array key cannot contain ']'") + default: + l.next() + } + } + return l.errorf("unclosed table array key") +} + +func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn { + for r := l.peek(); r != eof; r = l.peek() { + switch r { + case ']': + if len(l.buffer) > 0 { + l.emit(tokenKeyGroup) + } + l.next() + l.emit(tokenRightBracket) + return l.lexVoid + case '[': + return l.errorf("table key cannot contain ']'") + default: + l.next() + } + } + return l.errorf("unclosed table key") +} + +func (l *tomlLexer) lexRightBracket() tomlLexStateFn { + l.next() + l.emit(tokenRightBracket) + return l.lexRvalue +} + +func (l *tomlLexer) lexNumber() tomlLexStateFn { + r := l.peek() + if r == '+' || r == '-' { + l.next() + } + pointSeen := false + expSeen := false + digitSeen := false + for { + next := l.peek() + if next == '.' { + if pointSeen { + return l.errorf("cannot have two dots in one float") + } + l.next() + if !isDigit(l.peek()) { + return l.errorf("float cannot end with a dot") + } + pointSeen = true + } else if next == 'e' || next == 'E' { + expSeen = true + l.next() + r := l.peek() + if r == '+' || r == '-' { + l.next() + } + } else if isDigit(next) { + digitSeen = true + l.next() + } else if next == '_' { + l.next() + } else { + break + } + if pointSeen && !digitSeen { + return l.errorf("cannot start float with a dot") + } + } + + if !digitSeen { + return l.errorf("no digit in that number") + } + if pointSeen || expSeen { + l.emit(tokenFloat) + } else { + l.emit(tokenInteger) + } + return l.lexRvalue +} + +func (l *tomlLexer) run() { + for state := l.lexVoid; state != nil; { + state = state() + } + close(l.tokens) +} + +func init() { + dateRegexp = regexp.MustCompile(`^\d{1,4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,9})?(Z|[+-]\d{2}:\d{2})`) +} + +// Entry point +func lexToml(input io.Reader) chan token { + bufferedInput := buffruneio.NewReader(input) + l := &tomlLexer{ + input: bufferedInput, + tokens: make(chan token), + line: 1, + col: 1, + endbufferLine: 1, + endbufferCol: 1, + } + go l.run() + return l.tokens +} diff --git a/vendor/github.com/pelletier/go-toml/marshal.go b/vendor/github.com/pelletier/go-toml/marshal.go new file mode 100644 index 00000000..a1d70104 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/marshal.go @@ -0,0 +1,479 @@ +package toml + +import ( + "errors" + "fmt" + "reflect" + "strings" + "time" +) + +/* +TomlTree structural types and corresponding marshal types +------------------------------------------------------------------------------- +*TomlTree (*)struct, (*)map[string]interface{} +[]*TomlTree (*)[](*)struct, (*)[](*)map[string]interface{} +[]interface{} (as interface{}) (*)[]primitive, (*)[]([]interface{}) +interface{} (*)primitive + +TomlTree primitive types and corresponding marshal types +----------------------------------------------------------- +uint64 uint, uint8-uint64, pointers to same +int64 int, int8-uint64, pointers to same +float64 float32, float64, pointers to same +string string, pointers to same +bool bool, pointers to same +time.Time time.Time{}, pointers to same +*/ + +type tomlOpts struct { + name string + include bool + omitempty bool +} + +var timeType = reflect.TypeOf(time.Time{}) +var marshalerType = reflect.TypeOf(new(Marshaler)).Elem() + +// Check if the given marshall type maps to a TomlTree primitive +func isPrimitive(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Ptr: + return isPrimitive(mtype.Elem()) + case reflect.Bool: + return true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return true + case reflect.Float32, reflect.Float64: + return true + case reflect.String: + return true + case reflect.Struct: + return mtype == timeType || isCustomMarshaler(mtype) + default: + return false + } +} + +// Check if the given marshall type maps to a TomlTree slice +func isTreeSlice(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Slice: + return !isOtherSlice(mtype) + default: + return false + } +} + +// Check if the given marshall type maps to a non-TomlTree slice +func isOtherSlice(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Ptr: + return isOtherSlice(mtype.Elem()) + case reflect.Slice: + return isPrimitive(mtype.Elem()) || isOtherSlice(mtype.Elem()) + default: + return false + } +} + +// Check if the given marshall type maps to a TomlTree +func isTree(mtype reflect.Type) bool { + switch mtype.Kind() { + case reflect.Map: + return true + case reflect.Struct: + return !isPrimitive(mtype) + default: + return false + } +} + +func isCustomMarshaler(mtype reflect.Type) bool { + return mtype.Implements(marshalerType) +} + +func callCustomMarshaler(mval reflect.Value) ([]byte, error) { + return mval.Interface().(Marshaler).MarshalTOML() +} + +// Marshaler is the interface implemented by types that +// can marshal themselves into valid TOML. +type Marshaler interface { + MarshalTOML() ([]byte, error) +} + +/* +Marshal returns the TOML encoding of v. Behavior is similar to the Go json +encoder, except that there is no concept of a Marshaler interface or MarshalTOML +function for sub-structs, and currently only definite types can be marshaled +(i.e. no `interface{}`). + +Note that pointers are automatically assigned the "omitempty" option, as TOML +explicity does not handle null values (saying instead the label should be +dropped). +*/ +func Marshal(v interface{}) ([]byte, error) { + mtype := reflect.TypeOf(v) + if mtype.Kind() != reflect.Struct { + return []byte{}, errors.New("Only a struct can be marshaled to TOML") + } + sval := reflect.ValueOf(v) + if isCustomMarshaler(mtype) { + return callCustomMarshaler(sval) + } + t, err := valueToTree(mtype, sval) + if err != nil { + return []byte{}, err + } + s, err := t.ToTomlString() + return []byte(s), err +} + +// Convert given marshal struct or map value to toml tree +func valueToTree(mtype reflect.Type, mval reflect.Value) (*TomlTree, error) { + if mtype.Kind() == reflect.Ptr { + return valueToTree(mtype.Elem(), mval.Elem()) + } + tval := newTomlTree() + switch mtype.Kind() { + case reflect.Struct: + for i := 0; i < mtype.NumField(); i++ { + mtypef, mvalf := mtype.Field(i), mval.Field(i) + opts := tomlOptions(mtypef) + if opts.include && (!opts.omitempty || !isZero(mvalf)) { + val, err := valueToToml(mtypef.Type, mvalf) + if err != nil { + return nil, err + } + tval.Set(opts.name, val) + } + } + case reflect.Map: + for _, key := range mval.MapKeys() { + mvalf := mval.MapIndex(key) + val, err := valueToToml(mtype.Elem(), mvalf) + if err != nil { + return nil, err + } + tval.Set(key.String(), val) + } + } + return tval, nil +} + +// Convert given marshal slice to slice of Toml trees +func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*TomlTree, error) { + tval := make([]*TomlTree, mval.Len(), mval.Len()) + for i := 0; i < mval.Len(); i++ { + val, err := valueToTree(mtype.Elem(), mval.Index(i)) + if err != nil { + return nil, err + } + tval[i] = val + } + return tval, nil +} + +// Convert given marshal slice to slice of toml values +func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) { + tval := make([]interface{}, mval.Len(), mval.Len()) + for i := 0; i < mval.Len(); i++ { + val, err := valueToToml(mtype.Elem(), mval.Index(i)) + if err != nil { + return nil, err + } + tval[i] = val + } + return tval, nil +} + +// Convert given marshal value to toml value +func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { + if mtype.Kind() == reflect.Ptr { + return valueToToml(mtype.Elem(), mval.Elem()) + } + switch { + case isCustomMarshaler(mtype): + return callCustomMarshaler(mval) + case isTree(mtype): + return valueToTree(mtype, mval) + case isTreeSlice(mtype): + return valueToTreeSlice(mtype, mval) + case isOtherSlice(mtype): + return valueToOtherSlice(mtype, mval) + default: + switch mtype.Kind() { + case reflect.Bool: + return mval.Bool(), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return mval.Int(), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return mval.Uint(), nil + case reflect.Float32, reflect.Float64: + return mval.Float(), nil + case reflect.String: + return mval.String(), nil + case reflect.Struct: + return mval.Interface().(time.Time), nil + default: + return nil, fmt.Errorf("Marshal can't handle %v(%v)", mtype, mtype.Kind()) + } + } +} + +/* +Unmarshal parses the TOML-encoded data and stores the result in the value +pointed to by v. Behavior is similar to the Go json encoder, except that there +is no concept of an Unmarshaler interface or UnmarshalTOML function for +sub-structs, and currently only definite types can be unmarshaled to (i.e. no +`interface{}`). +*/ +func Unmarshal(data []byte, v interface{}) error { + mtype := reflect.TypeOf(v) + if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct { + return errors.New("Only a pointer to struct can be unmarshaled from TOML") + } + + t, err := Load(string(data)) + if err != nil { + return err + } + + sval, err := valueFromTree(mtype.Elem(), t) + if err != nil { + return err + } + reflect.ValueOf(v).Elem().Set(sval) + return nil +} + +// Convert toml tree to marshal struct or map, using marshal type +func valueFromTree(mtype reflect.Type, tval *TomlTree) (reflect.Value, error) { + if mtype.Kind() == reflect.Ptr { + return unwrapPointer(mtype, tval) + } + var mval reflect.Value + switch mtype.Kind() { + case reflect.Struct: + mval = reflect.New(mtype).Elem() + for i := 0; i < mtype.NumField(); i++ { + mtypef := mtype.Field(i) + opts := tomlOptions(mtypef) + if opts.include { + key := opts.name + exists := tval.Has(key) + if exists { + val := tval.Get(key) + mvalf, err := valueFromToml(mtypef.Type, val) + if err != nil { + return mval, formatError(err, tval.GetPosition(key)) + } + mval.Field(i).Set(mvalf) + } + } + } + case reflect.Map: + mval = reflect.MakeMap(mtype) + for _, key := range tval.Keys() { + val := tval.Get(key) + mvalf, err := valueFromToml(mtype.Elem(), val) + if err != nil { + return mval, formatError(err, tval.GetPosition(key)) + } + mval.SetMapIndex(reflect.ValueOf(key), mvalf) + } + } + return mval, nil +} + +// Convert toml value to marshal struct/map slice, using marshal type +func valueFromTreeSlice(mtype reflect.Type, tval []*TomlTree) (reflect.Value, error) { + mval := reflect.MakeSlice(mtype, len(tval), len(tval)) + for i := 0; i < len(tval); i++ { + val, err := valueFromTree(mtype.Elem(), tval[i]) + if err != nil { + return mval, err + } + mval.Index(i).Set(val) + } + return mval, nil +} + +// Convert toml value to marshal primitive slice, using marshal type +func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) { + mval := reflect.MakeSlice(mtype, len(tval), len(tval)) + for i := 0; i < len(tval); i++ { + val, err := valueFromToml(mtype.Elem(), tval[i]) + if err != nil { + return mval, err + } + mval.Index(i).Set(val) + } + return mval, nil +} + +// Convert toml value to marshal value, using marshal type +func valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) { + if mtype.Kind() == reflect.Ptr { + return unwrapPointer(mtype, tval) + } + switch { + case isTree(mtype): + return valueFromTree(mtype, tval.(*TomlTree)) + case isTreeSlice(mtype): + return valueFromTreeSlice(mtype, tval.([]*TomlTree)) + case isOtherSlice(mtype): + return valueFromOtherSlice(mtype, tval.([]interface{})) + default: + switch mtype.Kind() { + case reflect.Bool: + val, ok := tval.(bool) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to bool", tval, tval) + } + return reflect.ValueOf(val), nil + case reflect.Int: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) + } + return reflect.ValueOf(int(val)), nil + case reflect.Int8: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) + } + return reflect.ValueOf(int8(val)), nil + case reflect.Int16: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) + } + return reflect.ValueOf(int16(val)), nil + case reflect.Int32: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) + } + return reflect.ValueOf(int32(val)), nil + case reflect.Int64: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) + } + return reflect.ValueOf(val), nil + case reflect.Uint: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) + } + return reflect.ValueOf(uint(val)), nil + case reflect.Uint8: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) + } + return reflect.ValueOf(uint8(val)), nil + case reflect.Uint16: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) + } + return reflect.ValueOf(uint16(val)), nil + case reflect.Uint32: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) + } + return reflect.ValueOf(uint32(val)), nil + case reflect.Uint64: + val, ok := tval.(int64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) + } + return reflect.ValueOf(uint64(val)), nil + case reflect.Float32: + val, ok := tval.(float64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval) + } + return reflect.ValueOf(float32(val)), nil + case reflect.Float64: + val, ok := tval.(float64) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval) + } + return reflect.ValueOf(val), nil + case reflect.String: + val, ok := tval.(string) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to string", tval, tval) + } + return reflect.ValueOf(val), nil + case reflect.Struct: + val, ok := tval.(time.Time) + if !ok { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to time", tval, tval) + } + return reflect.ValueOf(val), nil + default: + return reflect.ValueOf(nil), fmt.Errorf("Unmarshal can't handle %v(%v)", mtype, mtype.Kind()) + } + } +} + +func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) { + val, err := valueFromToml(mtype.Elem(), tval) + if err != nil { + return reflect.ValueOf(nil), err + } + mval := reflect.New(mtype.Elem()) + mval.Elem().Set(val) + return mval, nil +} + +func tomlOptions(vf reflect.StructField) tomlOpts { + tag := vf.Tag.Get("toml") + parse := strings.Split(tag, ",") + result := tomlOpts{vf.Name, true, false} + if parse[0] != "" { + if parse[0] == "-" && len(parse) == 1 { + result.include = false + } else { + result.name = strings.Trim(parse[0], " ") + } + } + if vf.PkgPath != "" { + result.include = false + } + if len(parse) > 1 && strings.Trim(parse[1], " ") == "omitempty" { + result.omitempty = true + } + if vf.Type.Kind() == reflect.Ptr { + result.omitempty = true + } + return result +} + +func isZero(val reflect.Value) bool { + switch val.Type().Kind() { + case reflect.Map: + fallthrough + case reflect.Array: + fallthrough + case reflect.Slice: + return val.Len() == 0 + default: + return reflect.DeepEqual(val.Interface(), reflect.Zero(val.Type()).Interface()) + } +} + +func formatError(err error, pos Position) error { + if err.Error()[0] == '(' { // Error already contains position information + return err + } + return fmt.Errorf("%s: %s", pos, err) +} diff --git a/vendor/github.com/pelletier/go-toml/marshal_test.toml b/vendor/github.com/pelletier/go-toml/marshal_test.toml new file mode 100644 index 00000000..1c5f98e7 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/marshal_test.toml @@ -0,0 +1,38 @@ +title = "TOML Marshal Testing" + +[basic] + bool = true + date = 1979-05-27T07:32:00Z + float = 123.4 + int = 5000 + string = "Bite me" + uint = 5001 + +[basic_lists] + bools = [true,false,true] + dates = [1979-05-27T07:32:00Z,1980-05-27T07:32:00Z] + floats = [12.3,45.6,78.9] + ints = [8001,8001,8002] + strings = ["One","Two","Three"] + uints = [5002,5003] + +[basic_map] + one = "one" + two = "two" + +[subdoc] + + [subdoc.first] + name = "First" + + [subdoc.second] + name = "Second" + +[[subdoclist]] + name = "List.First" + +[[subdoclist]] + name = "List.Second" + +[[subdocptrs]] + name = "Second" diff --git a/vendor/github.com/pelletier/go-toml/match.go b/vendor/github.com/pelletier/go-toml/match.go new file mode 100644 index 00000000..48b0f2a1 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/match.go @@ -0,0 +1,234 @@ +package toml + +import ( + "fmt" +) + +// support function to set positions for tomlValues +// NOTE: this is done to allow ctx.lastPosition to indicate the start of any +// values returned by the query engines +func tomlValueCheck(node interface{}, ctx *queryContext) interface{} { + switch castNode := node.(type) { + case *tomlValue: + ctx.lastPosition = castNode.position + return castNode.value + case []*TomlTree: + if len(castNode) > 0 { + ctx.lastPosition = castNode[0].position + } + return node + default: + return node + } +} + +// base match +type matchBase struct { + next pathFn +} + +func (f *matchBase) setNext(next pathFn) { + f.next = next +} + +// terminating functor - gathers results +type terminatingFn struct { + // empty +} + +func newTerminatingFn() *terminatingFn { + return &terminatingFn{} +} + +func (f *terminatingFn) setNext(next pathFn) { + // do nothing +} + +func (f *terminatingFn) call(node interface{}, ctx *queryContext) { + switch castNode := node.(type) { + case *TomlTree: + ctx.result.appendResult(node, castNode.position) + case *tomlValue: + ctx.result.appendResult(node, castNode.position) + default: + // use last position for scalars + ctx.result.appendResult(node, ctx.lastPosition) + } +} + +// match single key +type matchKeyFn struct { + matchBase + Name string +} + +func newMatchKeyFn(name string) *matchKeyFn { + return &matchKeyFn{Name: name} +} + +func (f *matchKeyFn) call(node interface{}, ctx *queryContext) { + if array, ok := node.([]*TomlTree); ok { + for _, tree := range array { + item := tree.values[f.Name] + if item != nil { + f.next.call(item, ctx) + } + } + } else if tree, ok := node.(*TomlTree); ok { + item := tree.values[f.Name] + if item != nil { + f.next.call(item, ctx) + } + } +} + +// match single index +type matchIndexFn struct { + matchBase + Idx int +} + +func newMatchIndexFn(idx int) *matchIndexFn { + return &matchIndexFn{Idx: idx} +} + +func (f *matchIndexFn) call(node interface{}, ctx *queryContext) { + if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { + if f.Idx < len(arr) && f.Idx >= 0 { + f.next.call(arr[f.Idx], ctx) + } + } +} + +// filter by slicing +type matchSliceFn struct { + matchBase + Start, End, Step int +} + +func newMatchSliceFn(start, end, step int) *matchSliceFn { + return &matchSliceFn{Start: start, End: end, Step: step} +} + +func (f *matchSliceFn) call(node interface{}, ctx *queryContext) { + if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { + // adjust indexes for negative values, reverse ordering + realStart, realEnd := f.Start, f.End + if realStart < 0 { + realStart = len(arr) + realStart + } + if realEnd < 0 { + realEnd = len(arr) + realEnd + } + if realEnd < realStart { + realEnd, realStart = realStart, realEnd // swap + } + // loop and gather + for idx := realStart; idx < realEnd; idx += f.Step { + f.next.call(arr[idx], ctx) + } + } +} + +// match anything +type matchAnyFn struct { + matchBase +} + +func newMatchAnyFn() *matchAnyFn { + return &matchAnyFn{} +} + +func (f *matchAnyFn) call(node interface{}, ctx *queryContext) { + if tree, ok := node.(*TomlTree); ok { + for _, v := range tree.values { + f.next.call(v, ctx) + } + } +} + +// filter through union +type matchUnionFn struct { + Union []pathFn +} + +func (f *matchUnionFn) setNext(next pathFn) { + for _, fn := range f.Union { + fn.setNext(next) + } +} + +func (f *matchUnionFn) call(node interface{}, ctx *queryContext) { + for _, fn := range f.Union { + fn.call(node, ctx) + } +} + +// match every single last node in the tree +type matchRecursiveFn struct { + matchBase +} + +func newMatchRecursiveFn() *matchRecursiveFn { + return &matchRecursiveFn{} +} + +func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) { + if tree, ok := node.(*TomlTree); ok { + var visit func(tree *TomlTree) + visit = func(tree *TomlTree) { + for _, v := range tree.values { + f.next.call(v, ctx) + switch node := v.(type) { + case *TomlTree: + visit(node) + case []*TomlTree: + for _, subtree := range node { + visit(subtree) + } + } + } + } + f.next.call(tree, ctx) + visit(tree) + } +} + +// match based on an externally provided functional filter +type matchFilterFn struct { + matchBase + Pos Position + Name string +} + +func newMatchFilterFn(name string, pos Position) *matchFilterFn { + return &matchFilterFn{Name: name, Pos: pos} +} + +func (f *matchFilterFn) call(node interface{}, ctx *queryContext) { + fn, ok := (*ctx.filters)[f.Name] + if !ok { + panic(fmt.Sprintf("%s: query context does not have filter '%s'", + f.Pos.String(), f.Name)) + } + switch castNode := tomlValueCheck(node, ctx).(type) { + case *TomlTree: + for _, v := range castNode.values { + if tv, ok := v.(*tomlValue); ok { + if fn(tv.value) { + f.next.call(v, ctx) + } + } else { + if fn(v) { + f.next.call(v, ctx) + } + } + } + case []interface{}: + for _, v := range castNode { + if fn(v) { + f.next.call(v, ctx) + } + } + } +} diff --git a/vendor/github.com/pelletier/go-toml/parser.go b/vendor/github.com/pelletier/go-toml/parser.go new file mode 100644 index 00000000..20e90a3e --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/parser.go @@ -0,0 +1,393 @@ +// TOML Parser. + +package toml + +import ( + "errors" + "fmt" + "reflect" + "regexp" + "strconv" + "strings" + "time" +) + +type tomlParser struct { + flow chan token + tree *TomlTree + tokensBuffer []token + currentTable []string + seenTableKeys []string +} + +type tomlParserStateFn func() tomlParserStateFn + +// Formats and panics an error message based on a token +func (p *tomlParser) raiseError(tok *token, msg string, args ...interface{}) { + panic(tok.Position.String() + ": " + fmt.Sprintf(msg, args...)) +} + +func (p *tomlParser) run() { + for state := p.parseStart; state != nil; { + state = state() + } +} + +func (p *tomlParser) peek() *token { + if len(p.tokensBuffer) != 0 { + return &(p.tokensBuffer[0]) + } + + tok, ok := <-p.flow + if !ok { + return nil + } + p.tokensBuffer = append(p.tokensBuffer, tok) + return &tok +} + +func (p *tomlParser) assume(typ tokenType) { + tok := p.getToken() + if tok == nil { + p.raiseError(tok, "was expecting token %s, but token stream is empty", tok) + } + if tok.typ != typ { + p.raiseError(tok, "was expecting token %s, but got %s instead", typ, tok) + } +} + +func (p *tomlParser) getToken() *token { + if len(p.tokensBuffer) != 0 { + tok := p.tokensBuffer[0] + p.tokensBuffer = p.tokensBuffer[1:] + return &tok + } + tok, ok := <-p.flow + if !ok { + return nil + } + return &tok +} + +func (p *tomlParser) parseStart() tomlParserStateFn { + tok := p.peek() + + // end of stream, parsing is finished + if tok == nil { + return nil + } + + switch tok.typ { + case tokenDoubleLeftBracket: + return p.parseGroupArray + case tokenLeftBracket: + return p.parseGroup + case tokenKey: + return p.parseAssign + case tokenEOF: + return nil + default: + p.raiseError(tok, "unexpected token") + } + return nil +} + +func (p *tomlParser) parseGroupArray() tomlParserStateFn { + startToken := p.getToken() // discard the [[ + key := p.getToken() + if key.typ != tokenKeyGroupArray { + p.raiseError(key, "unexpected token %s, was expecting a table array key", key) + } + + // get or create table array element at the indicated part in the path + keys, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "invalid table array key: %s", err) + } + p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries + destTree := p.tree.GetPath(keys) + var array []*TomlTree + if destTree == nil { + array = make([]*TomlTree, 0) + } else if target, ok := destTree.([]*TomlTree); ok && target != nil { + array = destTree.([]*TomlTree) + } else { + p.raiseError(key, "key %s is already assigned and not of type table array", key) + } + p.currentTable = keys + + // add a new tree to the end of the table array + newTree := newTomlTree() + newTree.position = startToken.Position + array = append(array, newTree) + p.tree.SetPath(p.currentTable, array) + + // remove all keys that were children of this table array + prefix := key.val + "." + found := false + for ii := 0; ii < len(p.seenTableKeys); { + tableKey := p.seenTableKeys[ii] + if strings.HasPrefix(tableKey, prefix) { + p.seenTableKeys = append(p.seenTableKeys[:ii], p.seenTableKeys[ii+1:]...) + } else { + found = (tableKey == key.val) + ii++ + } + } + + // keep this key name from use by other kinds of assignments + if !found { + p.seenTableKeys = append(p.seenTableKeys, key.val) + } + + // move to next parser state + p.assume(tokenDoubleRightBracket) + return p.parseStart +} + +func (p *tomlParser) parseGroup() tomlParserStateFn { + startToken := p.getToken() // discard the [ + key := p.getToken() + if key.typ != tokenKeyGroup { + p.raiseError(key, "unexpected token %s, was expecting a table key", key) + } + for _, item := range p.seenTableKeys { + if item == key.val { + p.raiseError(key, "duplicated tables") + } + } + + p.seenTableKeys = append(p.seenTableKeys, key.val) + keys, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "invalid table array key: %s", err) + } + if err := p.tree.createSubTree(keys, startToken.Position); err != nil { + p.raiseError(key, "%s", err) + } + p.assume(tokenRightBracket) + p.currentTable = keys + return p.parseStart +} + +func (p *tomlParser) parseAssign() tomlParserStateFn { + key := p.getToken() + p.assume(tokenEqual) + + value := p.parseRvalue() + var tableKey []string + if len(p.currentTable) > 0 { + tableKey = p.currentTable + } else { + tableKey = []string{} + } + + // find the table to assign, looking out for arrays of tables + var targetNode *TomlTree + switch node := p.tree.GetPath(tableKey).(type) { + case []*TomlTree: + targetNode = node[len(node)-1] + case *TomlTree: + targetNode = node + default: + p.raiseError(key, "Unknown table type for path: %s", + strings.Join(tableKey, ".")) + } + + // assign value to the found table + keyVals, err := parseKey(key.val) + if err != nil { + p.raiseError(key, "%s", err) + } + if len(keyVals) != 1 { + p.raiseError(key, "Invalid key") + } + keyVal := keyVals[0] + localKey := []string{keyVal} + finalKey := append(tableKey, keyVal) + if targetNode.GetPath(localKey) != nil { + p.raiseError(key, "The following key was defined twice: %s", + strings.Join(finalKey, ".")) + } + var toInsert interface{} + + switch value.(type) { + case *TomlTree, []*TomlTree: + toInsert = value + default: + toInsert = &tomlValue{value, key.Position} + } + targetNode.values[keyVal] = toInsert + return p.parseStart +} + +var numberUnderscoreInvalidRegexp *regexp.Regexp + +func cleanupNumberToken(value string) (string, error) { + if numberUnderscoreInvalidRegexp.MatchString(value) { + return "", errors.New("invalid use of _ in number") + } + cleanedVal := strings.Replace(value, "_", "", -1) + return cleanedVal, nil +} + +func (p *tomlParser) parseRvalue() interface{} { + tok := p.getToken() + if tok == nil || tok.typ == tokenEOF { + p.raiseError(tok, "expecting a value") + } + + switch tok.typ { + case tokenString: + return tok.val + case tokenTrue: + return true + case tokenFalse: + return false + case tokenInteger: + cleanedVal, err := cleanupNumberToken(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err := strconv.ParseInt(cleanedVal, 10, 64) + if err != nil { + p.raiseError(tok, "%s", err) + } + return val + case tokenFloat: + cleanedVal, err := cleanupNumberToken(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err := strconv.ParseFloat(cleanedVal, 64) + if err != nil { + p.raiseError(tok, "%s", err) + } + return val + case tokenDate: + val, err := time.ParseInLocation(time.RFC3339Nano, tok.val, time.UTC) + if err != nil { + p.raiseError(tok, "%s", err) + } + return val + case tokenLeftBracket: + return p.parseArray() + case tokenLeftCurlyBrace: + return p.parseInlineTable() + case tokenEqual: + p.raiseError(tok, "cannot have multiple equals for the same key") + case tokenError: + p.raiseError(tok, "%s", tok) + } + + p.raiseError(tok, "never reached") + + return nil +} + +func tokenIsComma(t *token) bool { + return t != nil && t.typ == tokenComma +} + +func (p *tomlParser) parseInlineTable() *TomlTree { + tree := newTomlTree() + var previous *token +Loop: + for { + follow := p.peek() + if follow == nil || follow.typ == tokenEOF { + p.raiseError(follow, "unterminated inline table") + } + switch follow.typ { + case tokenRightCurlyBrace: + p.getToken() + break Loop + case tokenKey: + if !tokenIsComma(previous) && previous != nil { + p.raiseError(follow, "comma expected between fields in inline table") + } + key := p.getToken() + p.assume(tokenEqual) + value := p.parseRvalue() + tree.Set(key.val, value) + case tokenComma: + if previous == nil { + p.raiseError(follow, "inline table cannot start with a comma") + } + if tokenIsComma(previous) { + p.raiseError(follow, "need field between two commas in inline table") + } + p.getToken() + default: + p.raiseError(follow, "unexpected token type in inline table: %s", follow.typ.String()) + } + previous = follow + } + if tokenIsComma(previous) { + p.raiseError(previous, "trailing comma at the end of inline table") + } + return tree +} + +func (p *tomlParser) parseArray() interface{} { + var array []interface{} + arrayType := reflect.TypeOf(nil) + for { + follow := p.peek() + if follow == nil || follow.typ == tokenEOF { + p.raiseError(follow, "unterminated array") + } + if follow.typ == tokenRightBracket { + p.getToken() + break + } + val := p.parseRvalue() + if arrayType == nil { + arrayType = reflect.TypeOf(val) + } + if reflect.TypeOf(val) != arrayType { + p.raiseError(follow, "mixed types in array") + } + array = append(array, val) + follow = p.peek() + if follow == nil || follow.typ == tokenEOF { + p.raiseError(follow, "unterminated array") + } + if follow.typ != tokenRightBracket && follow.typ != tokenComma { + p.raiseError(follow, "missing comma") + } + if follow.typ == tokenComma { + p.getToken() + } + } + // An array of TomlTrees is actually an array of inline + // tables, which is a shorthand for a table array. If the + // array was not converted from []interface{} to []*TomlTree, + // the two notations would not be equivalent. + if arrayType == reflect.TypeOf(newTomlTree()) { + tomlArray := make([]*TomlTree, len(array)) + for i, v := range array { + tomlArray[i] = v.(*TomlTree) + } + return tomlArray + } + return array +} + +func parseToml(flow chan token) *TomlTree { + result := newTomlTree() + result.position = Position{1, 1} + parser := &tomlParser{ + flow: flow, + tree: result, + tokensBuffer: make([]token, 0), + currentTable: make([]string, 0), + seenTableKeys: make([]string, 0), + } + parser.run() + return result +} + +func init() { + numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d]|_$|^_)`) +} diff --git a/vendor/github.com/pelletier/go-toml/position.go b/vendor/github.com/pelletier/go-toml/position.go new file mode 100644 index 00000000..c17bff87 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/position.go @@ -0,0 +1,29 @@ +// Position support for go-toml + +package toml + +import ( + "fmt" +) + +// Position of a document element within a TOML document. +// +// Line and Col are both 1-indexed positions for the element's line number and +// column number, respectively. Values of zero or less will cause Invalid(), +// to return true. +type Position struct { + Line int // line within the document + Col int // column within the line +} + +// String representation of the position. +// Displays 1-indexed line and column numbers. +func (p Position) String() string { + return fmt.Sprintf("(%d, %d)", p.Line, p.Col) +} + +// Invalid returns whether or not the position is valid (i.e. with negative or +// null values) +func (p Position) Invalid() bool { + return p.Line <= 0 || p.Col <= 0 +} diff --git a/vendor/github.com/pelletier/go-toml/query.go b/vendor/github.com/pelletier/go-toml/query.go new file mode 100644 index 00000000..307a1eca --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/query.go @@ -0,0 +1,153 @@ +package toml + +import ( + "time" +) + +// NodeFilterFn represents a user-defined filter function, for use with +// Query.SetFilter(). +// +// The return value of the function must indicate if 'node' is to be included +// at this stage of the TOML path. Returning true will include the node, and +// returning false will exclude it. +// +// NOTE: Care should be taken to write script callbacks such that they are safe +// to use from multiple goroutines. +type NodeFilterFn func(node interface{}) bool + +// QueryResult is the result of Executing a Query. +type QueryResult struct { + items []interface{} + positions []Position +} + +// appends a value/position pair to the result set. +func (r *QueryResult) appendResult(node interface{}, pos Position) { + r.items = append(r.items, node) + r.positions = append(r.positions, pos) +} + +// Values is a set of values within a QueryResult. The order of values is not +// guaranteed to be in document order, and may be different each time a query is +// executed. +func (r QueryResult) Values() []interface{} { + values := make([]interface{}, len(r.items)) + for i, v := range r.items { + o, ok := v.(*tomlValue) + if ok { + values[i] = o.value + } else { + values[i] = v + } + } + return values +} + +// Positions is a set of positions for values within a QueryResult. Each index +// in Positions() corresponds to the entry in Value() of the same index. +func (r QueryResult) Positions() []Position { + return r.positions +} + +// runtime context for executing query paths +type queryContext struct { + result *QueryResult + filters *map[string]NodeFilterFn + lastPosition Position +} + +// generic path functor interface +type pathFn interface { + setNext(next pathFn) + call(node interface{}, ctx *queryContext) +} + +// A Query is the representation of a compiled TOML path. A Query is safe +// for concurrent use by multiple goroutines. +type Query struct { + root pathFn + tail pathFn + filters *map[string]NodeFilterFn +} + +func newQuery() *Query { + return &Query{ + root: nil, + tail: nil, + filters: &defaultFilterFunctions, + } +} + +func (q *Query) appendPath(next pathFn) { + if q.root == nil { + q.root = next + } else { + q.tail.setNext(next) + } + q.tail = next + next.setNext(newTerminatingFn()) // init the next functor +} + +// CompileQuery compiles a TOML path expression. The returned Query can be used +// to match elements within a TomlTree and its descendants. +func CompileQuery(path string) (*Query, error) { + return parseQuery(lexQuery(path)) +} + +// Execute executes a query against a TomlTree, and returns the result of the query. +func (q *Query) Execute(tree *TomlTree) *QueryResult { + result := &QueryResult{ + items: []interface{}{}, + positions: []Position{}, + } + if q.root == nil { + result.appendResult(tree, tree.GetPosition("")) + } else { + ctx := &queryContext{ + result: result, + filters: q.filters, + } + q.root.call(tree, ctx) + } + return result +} + +// SetFilter sets a user-defined filter function. These may be used inside +// "?(..)" query expressions to filter TOML document elements within a query. +func (q *Query) SetFilter(name string, fn NodeFilterFn) { + if q.filters == &defaultFilterFunctions { + // clone the static table + q.filters = &map[string]NodeFilterFn{} + for k, v := range defaultFilterFunctions { + (*q.filters)[k] = v + } + } + (*q.filters)[name] = fn +} + +var defaultFilterFunctions = map[string]NodeFilterFn{ + "tree": func(node interface{}) bool { + _, ok := node.(*TomlTree) + return ok + }, + "int": func(node interface{}) bool { + _, ok := node.(int64) + return ok + }, + "float": func(node interface{}) bool { + _, ok := node.(float64) + return ok + }, + "string": func(node interface{}) bool { + _, ok := node.(string) + return ok + }, + "time": func(node interface{}) bool { + _, ok := node.(time.Time) + return ok + }, + "bool": func(node interface{}) bool { + _, ok := node.(bool) + return ok + }, +} diff --git a/vendor/github.com/pelletier/go-toml/querylexer.go b/vendor/github.com/pelletier/go-toml/querylexer.go new file mode 100644 index 00000000..960681d0 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/querylexer.go @@ -0,0 +1,356 @@ +// TOML JSONPath lexer. +// +// Written using the principles developed by Rob Pike in +// http://www.youtube.com/watch?v=HxaD_trXwRE + +package toml + +import ( + "fmt" + "strconv" + "strings" + "unicode/utf8" +) + +// Lexer state function +type queryLexStateFn func() queryLexStateFn + +// Lexer definition +type queryLexer struct { + input string + start int + pos int + width int + tokens chan token + depth int + line int + col int + stringTerm string +} + +func (l *queryLexer) run() { + for state := l.lexVoid; state != nil; { + state = state() + } + close(l.tokens) +} + +func (l *queryLexer) nextStart() { + // iterate by runes (utf8 characters) + // search for newlines and advance line/col counts + for i := l.start; i < l.pos; { + r, width := utf8.DecodeRuneInString(l.input[i:]) + if r == '\n' { + l.line++ + l.col = 1 + } else { + l.col++ + } + i += width + } + // advance start position to next token + l.start = l.pos +} + +func (l *queryLexer) emit(t tokenType) { + l.tokens <- token{ + Position: Position{l.line, l.col}, + typ: t, + val: l.input[l.start:l.pos], + } + l.nextStart() +} + +func (l *queryLexer) emitWithValue(t tokenType, value string) { + l.tokens <- token{ + Position: Position{l.line, l.col}, + typ: t, + val: value, + } + l.nextStart() +} + +func (l *queryLexer) next() rune { + if l.pos >= len(l.input) { + l.width = 0 + return eof + } + var r rune + r, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) + l.pos += l.width + return r +} + +func (l *queryLexer) ignore() { + l.nextStart() +} + +func (l *queryLexer) backup() { + l.pos -= l.width +} + +func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn { + l.tokens <- token{ + Position: Position{l.line, l.col}, + typ: tokenError, + val: fmt.Sprintf(format, args...), + } + return nil +} + +func (l *queryLexer) peek() rune { + r := l.next() + l.backup() + return r +} + +func (l *queryLexer) accept(valid string) bool { + if strings.ContainsRune(valid, l.next()) { + return true + } + l.backup() + return false +} + +func (l *queryLexer) follow(next string) bool { + return strings.HasPrefix(l.input[l.pos:], next) +} + +func (l *queryLexer) lexVoid() queryLexStateFn { + for { + next := l.peek() + switch next { + case '$': + l.pos++ + l.emit(tokenDollar) + continue + case '.': + if l.follow("..") { + l.pos += 2 + l.emit(tokenDotDot) + } else { + l.pos++ + l.emit(tokenDot) + } + continue + case '[': + l.pos++ + l.emit(tokenLeftBracket) + continue + case ']': + l.pos++ + l.emit(tokenRightBracket) + continue + case ',': + l.pos++ + l.emit(tokenComma) + continue + case '*': + l.pos++ + l.emit(tokenStar) + continue + case '(': + l.pos++ + l.emit(tokenLeftParen) + continue + case ')': + l.pos++ + l.emit(tokenRightParen) + continue + case '?': + l.pos++ + l.emit(tokenQuestion) + continue + case ':': + l.pos++ + l.emit(tokenColon) + continue + case '\'': + l.ignore() + l.stringTerm = string(next) + return l.lexString + case '"': + l.ignore() + l.stringTerm = string(next) + return l.lexString + } + + if isSpace(next) { + l.next() + l.ignore() + continue + } + + if isAlphanumeric(next) { + return l.lexKey + } + + if next == '+' || next == '-' || isDigit(next) { + return l.lexNumber + } + + if l.next() == eof { + break + } + + return l.errorf("unexpected char: '%v'", next) + } + l.emit(tokenEOF) + return nil +} + +func (l *queryLexer) lexKey() queryLexStateFn { + for { + next := l.peek() + if !isAlphanumeric(next) { + l.emit(tokenKey) + return l.lexVoid + } + + if l.next() == eof { + break + } + } + l.emit(tokenEOF) + return nil +} + +func (l *queryLexer) lexString() queryLexStateFn { + l.pos++ + l.ignore() + growingString := "" + + for { + if l.follow(l.stringTerm) { + l.emitWithValue(tokenString, growingString) + l.pos++ + l.ignore() + return l.lexVoid + } + + if l.follow("\\\"") { + l.pos++ + growingString += "\"" + } else if l.follow("\\'") { + l.pos++ + growingString += "'" + } else if l.follow("\\n") { + l.pos++ + growingString += "\n" + } else if l.follow("\\b") { + l.pos++ + growingString += "\b" + } else if l.follow("\\f") { + l.pos++ + growingString += "\f" + } else if l.follow("\\/") { + l.pos++ + growingString += "/" + } else if l.follow("\\t") { + l.pos++ + growingString += "\t" + } else if l.follow("\\r") { + l.pos++ + growingString += "\r" + } else if l.follow("\\\\") { + l.pos++ + growingString += "\\" + } else if l.follow("\\u") { + l.pos += 2 + code := "" + for i := 0; i < 4; i++ { + c := l.peek() + l.pos++ + if !isHexDigit(c) { + return l.errorf("unfinished unicode escape") + } + code = code + string(c) + } + l.pos-- + intcode, err := strconv.ParseInt(code, 16, 32) + if err != nil { + return l.errorf("invalid unicode escape: \\u" + code) + } + growingString += string(rune(intcode)) + } else if l.follow("\\U") { + l.pos += 2 + code := "" + for i := 0; i < 8; i++ { + c := l.peek() + l.pos++ + if !isHexDigit(c) { + return l.errorf("unfinished unicode escape") + } + code = code + string(c) + } + l.pos-- + intcode, err := strconv.ParseInt(code, 16, 32) + if err != nil { + return l.errorf("invalid unicode escape: \\u" + code) + } + growingString += string(rune(intcode)) + } else if l.follow("\\") { + l.pos++ + return l.errorf("invalid escape sequence: \\" + string(l.peek())) + } else { + growingString += string(l.peek()) + } + + if l.next() == eof { + break + } + } + + return l.errorf("unclosed string") +} + +func (l *queryLexer) lexNumber() queryLexStateFn { + l.ignore() + if !l.accept("+") { + l.accept("-") + } + pointSeen := false + digitSeen := false + for { + next := l.next() + if next == '.' { + if pointSeen { + return l.errorf("cannot have two dots in one float") + } + if !isDigit(l.peek()) { + return l.errorf("float cannot end with a dot") + } + pointSeen = true + } else if isDigit(next) { + digitSeen = true + } else { + l.backup() + break + } + if pointSeen && !digitSeen { + return l.errorf("cannot start float with a dot") + } + } + + if !digitSeen { + return l.errorf("no digit in that number") + } + if pointSeen { + l.emit(tokenFloat) + } else { + l.emit(tokenInteger) + } + return l.lexVoid +} + +// Entry point +func lexQuery(input string) chan token { + l := &queryLexer{ + input: input, + tokens: make(chan token), + line: 1, + col: 1, + } + go l.run() + return l.tokens +} diff --git a/vendor/github.com/pelletier/go-toml/queryparser.go b/vendor/github.com/pelletier/go-toml/queryparser.go new file mode 100644 index 00000000..1cbfc83b --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/queryparser.go @@ -0,0 +1,275 @@ +/* + Based on the "jsonpath" spec/concept. + + http://goessner.net/articles/JsonPath/ + https://code.google.com/p/json-path/ +*/ + +package toml + +import ( + "fmt" +) + +const maxInt = int(^uint(0) >> 1) + +type queryParser struct { + flow chan token + tokensBuffer []token + query *Query + union []pathFn + err error +} + +type queryParserStateFn func() queryParserStateFn + +// Formats and panics an error message based on a token +func (p *queryParser) parseError(tok *token, msg string, args ...interface{}) queryParserStateFn { + p.err = fmt.Errorf(tok.Position.String()+": "+msg, args...) + return nil // trigger parse to end +} + +func (p *queryParser) run() { + for state := p.parseStart; state != nil; { + state = state() + } +} + +func (p *queryParser) backup(tok *token) { + p.tokensBuffer = append(p.tokensBuffer, *tok) +} + +func (p *queryParser) peek() *token { + if len(p.tokensBuffer) != 0 { + return &(p.tokensBuffer[0]) + } + + tok, ok := <-p.flow + if !ok { + return nil + } + p.backup(&tok) + return &tok +} + +func (p *queryParser) lookahead(types ...tokenType) bool { + result := true + buffer := []token{} + + for _, typ := range types { + tok := p.getToken() + if tok == nil { + result = false + break + } + buffer = append(buffer, *tok) + if tok.typ != typ { + result = false + break + } + } + // add the tokens back to the buffer, and return + p.tokensBuffer = append(p.tokensBuffer, buffer...) + return result +} + +func (p *queryParser) getToken() *token { + if len(p.tokensBuffer) != 0 { + tok := p.tokensBuffer[0] + p.tokensBuffer = p.tokensBuffer[1:] + return &tok + } + tok, ok := <-p.flow + if !ok { + return nil + } + return &tok +} + +func (p *queryParser) parseStart() queryParserStateFn { + tok := p.getToken() + + if tok == nil || tok.typ == tokenEOF { + return nil + } + + if tok.typ != tokenDollar { + return p.parseError(tok, "Expected '$' at start of expression") + } + + return p.parseMatchExpr +} + +// handle '.' prefix, '[]', and '..' +func (p *queryParser) parseMatchExpr() queryParserStateFn { + tok := p.getToken() + switch tok.typ { + case tokenDotDot: + p.query.appendPath(&matchRecursiveFn{}) + // nested parse for '..' + tok := p.getToken() + switch tok.typ { + case tokenKey: + p.query.appendPath(newMatchKeyFn(tok.val)) + return p.parseMatchExpr + case tokenLeftBracket: + return p.parseBracketExpr + case tokenStar: + // do nothing - the recursive predicate is enough + return p.parseMatchExpr + } + + case tokenDot: + // nested parse for '.' + tok := p.getToken() + switch tok.typ { + case tokenKey: + p.query.appendPath(newMatchKeyFn(tok.val)) + return p.parseMatchExpr + case tokenStar: + p.query.appendPath(&matchAnyFn{}) + return p.parseMatchExpr + } + + case tokenLeftBracket: + return p.parseBracketExpr + + case tokenEOF: + return nil // allow EOF at this stage + } + return p.parseError(tok, "expected match expression") +} + +func (p *queryParser) parseBracketExpr() queryParserStateFn { + if p.lookahead(tokenInteger, tokenColon) { + return p.parseSliceExpr + } + if p.peek().typ == tokenColon { + return p.parseSliceExpr + } + return p.parseUnionExpr +} + +func (p *queryParser) parseUnionExpr() queryParserStateFn { + var tok *token + + // this state can be traversed after some sub-expressions + // so be careful when setting up state in the parser + if p.union == nil { + p.union = []pathFn{} + } + +loop: // labeled loop for easy breaking + for { + if len(p.union) > 0 { + // parse delimiter or terminator + tok = p.getToken() + switch tok.typ { + case tokenComma: + // do nothing + case tokenRightBracket: + break loop + default: + return p.parseError(tok, "expected ',' or ']', not '%s'", tok.val) + } + } + + // parse sub expression + tok = p.getToken() + switch tok.typ { + case tokenInteger: + p.union = append(p.union, newMatchIndexFn(tok.Int())) + case tokenKey: + p.union = append(p.union, newMatchKeyFn(tok.val)) + case tokenString: + p.union = append(p.union, newMatchKeyFn(tok.val)) + case tokenQuestion: + return p.parseFilterExpr + default: + return p.parseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union)) + } + } + + // if there is only one sub-expression, use that instead + if len(p.union) == 1 { + p.query.appendPath(p.union[0]) + } else { + p.query.appendPath(&matchUnionFn{p.union}) + } + + p.union = nil // clear out state + return p.parseMatchExpr +} + +func (p *queryParser) parseSliceExpr() queryParserStateFn { + // init slice to grab all elements + start, end, step := 0, maxInt, 1 + + // parse optional start + tok := p.getToken() + if tok.typ == tokenInteger { + start = tok.Int() + tok = p.getToken() + } + if tok.typ != tokenColon { + return p.parseError(tok, "expected ':'") + } + + // parse optional end + tok = p.getToken() + if tok.typ == tokenInteger { + end = tok.Int() + tok = p.getToken() + } + if tok.typ == tokenRightBracket { + p.query.appendPath(newMatchSliceFn(start, end, step)) + return p.parseMatchExpr + } + if tok.typ != tokenColon { + return p.parseError(tok, "expected ']' or ':'") + } + + // parse optional step + tok = p.getToken() + if tok.typ == tokenInteger { + step = tok.Int() + if step < 0 { + return p.parseError(tok, "step must be a positive value") + } + tok = p.getToken() + } + if tok.typ != tokenRightBracket { + return p.parseError(tok, "expected ']'") + } + + p.query.appendPath(newMatchSliceFn(start, end, step)) + return p.parseMatchExpr +} + +func (p *queryParser) parseFilterExpr() queryParserStateFn { + tok := p.getToken() + if tok.typ != tokenLeftParen { + return p.parseError(tok, "expected left-parenthesis for filter expression") + } + tok = p.getToken() + if tok.typ != tokenKey && tok.typ != tokenString { + return p.parseError(tok, "expected key or string for filter funciton name") + } + name := tok.val + tok = p.getToken() + if tok.typ != tokenRightParen { + return p.parseError(tok, "expected right-parenthesis for filter expression") + } + p.union = append(p.union, newMatchFilterFn(name, tok.Position)) + return p.parseUnionExpr +} + +func parseQuery(flow chan token) (*Query, error) { + parser := &queryParser{ + flow: flow, + tokensBuffer: []token{}, + query: newQuery(), + } + parser.run() + return parser.query, parser.err +} diff --git a/vendor/github.com/pelletier/go-toml/test.sh b/vendor/github.com/pelletier/go-toml/test.sh new file mode 100755 index 00000000..436d2fb6 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/test.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# fail out of the script if anything here fails +set -e + +# set the path to the present working directory +export GOPATH=`pwd` + +function git_clone() { + path=$1 + branch=$2 + version=$3 + if [ ! -d "src/$path" ]; then + mkdir -p src/$path + git clone https://$path.git src/$path + fi + pushd src/$path + git checkout "$branch" + git reset --hard "$version" + popd +} + +# Run go vet +go vet ./... + +go get github.com/pelletier/go-buffruneio +go get github.com/davecgh/go-spew/spew + +# get code for BurntSushi TOML validation +# pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize) +git_clone github.com/BurntSushi/toml master HEAD +git_clone github.com/BurntSushi/toml-test master HEAD #was: 0.2.0 HEAD + +# build the BurntSushi test application +go build -o toml-test github.com/BurntSushi/toml-test + +# vendorize the current lib for testing +# NOTE: this basically mocks an install without having to go back out to github for code +mkdir -p src/github.com/pelletier/go-toml/cmd +cp *.go *.toml src/github.com/pelletier/go-toml +cp -R cmd/* src/github.com/pelletier/go-toml/cmd +go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go + +# Run basic unit tests +go test github.com/pelletier/go-toml -v -covermode=count -coverprofile=coverage.out +go test github.com/pelletier/go-toml/cmd/tomljson + +# run the entire BurntSushi test suite +if [[ $# -eq 0 ]] ; then + echo "Running all BurntSushi tests" + ./toml-test ./test_program_bin | tee test_out +else + # run a specific test + test=$1 + test_path='src/github.com/BurntSushi/toml-test/tests' + valid_test="$test_path/valid/$test" + invalid_test="$test_path/invalid/$test" + + if [ -e "$valid_test.toml" ]; then + echo "Valid Test TOML for $test:" + echo "====" + cat "$valid_test.toml" + + echo "Valid Test JSON for $test:" + echo "====" + cat "$valid_test.json" + + echo "Go-TOML Output for $test:" + echo "====" + cat "$valid_test.toml" | ./test_program_bin + fi + + if [ -e "$invalid_test.toml" ]; then + echo "Invalid Test TOML for $test:" + echo "====" + cat "$invalid_test.toml" + + echo "Go-TOML Output for $test:" + echo "====" + echo "go-toml Output:" + cat "$invalid_test.toml" | ./test_program_bin + fi +fi diff --git a/vendor/github.com/pelletier/go-toml/token.go b/vendor/github.com/pelletier/go-toml/token.go new file mode 100644 index 00000000..5581fe0b --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/token.go @@ -0,0 +1,140 @@ +package toml + +import ( + "fmt" + "strconv" + "unicode" +) + +// Define tokens +type tokenType int + +const ( + eof = -(iota + 1) +) + +const ( + tokenError tokenType = iota + tokenEOF + tokenComment + tokenKey + tokenString + tokenInteger + tokenTrue + tokenFalse + tokenFloat + tokenEqual + tokenLeftBracket + tokenRightBracket + tokenLeftCurlyBrace + tokenRightCurlyBrace + tokenLeftParen + tokenRightParen + tokenDoubleLeftBracket + tokenDoubleRightBracket + tokenDate + tokenKeyGroup + tokenKeyGroupArray + tokenComma + tokenColon + tokenDollar + tokenStar + tokenQuestion + tokenDot + tokenDotDot + tokenEOL +) + +var tokenTypeNames = []string{ + "Error", + "EOF", + "Comment", + "Key", + "String", + "Integer", + "True", + "False", + "Float", + "=", + "[", + "]", + "{", + "}", + "(", + ")", + "]]", + "[[", + "Date", + "KeyGroup", + "KeyGroupArray", + ",", + ":", + "$", + "*", + "?", + ".", + "..", + "EOL", +} + +type token struct { + Position + typ tokenType + val string +} + +func (tt tokenType) String() string { + idx := int(tt) + if idx < len(tokenTypeNames) { + return tokenTypeNames[idx] + } + return "Unknown" +} + +func (t token) Int() int { + if result, err := strconv.Atoi(t.val); err != nil { + panic(err) + } else { + return result + } +} + +func (t token) String() string { + switch t.typ { + case tokenEOF: + return "EOF" + case tokenError: + return t.val + } + + return fmt.Sprintf("%q", t.val) +} + +func isSpace(r rune) bool { + return r == ' ' || r == '\t' +} + +func isAlphanumeric(r rune) bool { + return unicode.IsLetter(r) || r == '_' +} + +func isKeyChar(r rune) bool { + // Keys start with the first character that isn't whitespace or [ and end + // with the last non-whitespace character before the equals sign. Keys + // cannot contain a # character." + return !(r == '\r' || r == '\n' || r == eof || r == '=') +} + +func isKeyStartChar(r rune) bool { + return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[') +} + +func isDigit(r rune) bool { + return unicode.IsNumber(r) +} + +func isHexDigit(r rune) bool { + return isDigit(r) || + (r >= 'a' && r <= 'f') || + (r >= 'A' && r <= 'F') +} diff --git a/vendor/github.com/pelletier/go-toml/toml.go b/vendor/github.com/pelletier/go-toml/toml.go new file mode 100644 index 00000000..1ba56a1c --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/toml.go @@ -0,0 +1,284 @@ +package toml + +import ( + "errors" + "fmt" + "io" + "os" + "runtime" + "strings" +) + +type tomlValue struct { + value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list + position Position +} + +// TomlTree is the result of the parsing of a TOML file. +type TomlTree struct { + values map[string]interface{} // string -> *tomlValue, *TomlTree, []*TomlTree + position Position +} + +func newTomlTree() *TomlTree { + return &TomlTree{ + values: make(map[string]interface{}), + position: Position{}, + } +} + +// TreeFromMap initializes a new TomlTree object using the given map. +func TreeFromMap(m map[string]interface{}) (*TomlTree, error) { + result, err := toTree(m) + if err != nil { + return nil, err + } + return result.(*TomlTree), nil +} + +// Has returns a boolean indicating if the given key exists. +func (t *TomlTree) Has(key string) bool { + if key == "" { + return false + } + return t.HasPath(strings.Split(key, ".")) +} + +// HasPath returns true if the given path of keys exists, false otherwise. +func (t *TomlTree) HasPath(keys []string) bool { + return t.GetPath(keys) != nil +} + +// Keys returns the keys of the toplevel tree. +// Warning: this is a costly operation. +func (t *TomlTree) Keys() []string { + var keys []string + for k := range t.values { + keys = append(keys, k) + } + return keys +} + +// Get the value at key in the TomlTree. +// Key is a dot-separated path (e.g. a.b.c). +// Returns nil if the path does not exist in the tree. +// If keys is of length zero, the current tree is returned. +func (t *TomlTree) Get(key string) interface{} { + if key == "" { + return t + } + comps, err := parseKey(key) + if err != nil { + return nil + } + return t.GetPath(comps) +} + +// GetPath returns the element in the tree indicated by 'keys'. +// If keys is of length zero, the current tree is returned. +func (t *TomlTree) GetPath(keys []string) interface{} { + if len(keys) == 0 { + return t + } + subtree := t + for _, intermediateKey := range keys[:len(keys)-1] { + value, exists := subtree.values[intermediateKey] + if !exists { + return nil + } + switch node := value.(type) { + case *TomlTree: + subtree = node + case []*TomlTree: + // go to most recent element + if len(node) == 0 { + return nil + } + subtree = node[len(node)-1] + default: + return nil // cannot navigate through other node types + } + } + // branch based on final node type + switch node := subtree.values[keys[len(keys)-1]].(type) { + case *tomlValue: + return node.value + default: + return node + } +} + +// GetPosition returns the position of the given key. +func (t *TomlTree) GetPosition(key string) Position { + if key == "" { + return t.position + } + return t.GetPositionPath(strings.Split(key, ".")) +} + +// GetPositionPath returns the element in the tree indicated by 'keys'. +// If keys is of length zero, the current tree is returned. +func (t *TomlTree) GetPositionPath(keys []string) Position { + if len(keys) == 0 { + return t.position + } + subtree := t + for _, intermediateKey := range keys[:len(keys)-1] { + value, exists := subtree.values[intermediateKey] + if !exists { + return Position{0, 0} + } + switch node := value.(type) { + case *TomlTree: + subtree = node + case []*TomlTree: + // go to most recent element + if len(node) == 0 { + return Position{0, 0} + } + subtree = node[len(node)-1] + default: + return Position{0, 0} + } + } + // branch based on final node type + switch node := subtree.values[keys[len(keys)-1]].(type) { + case *tomlValue: + return node.position + case *TomlTree: + return node.position + case []*TomlTree: + // go to most recent element + if len(node) == 0 { + return Position{0, 0} + } + return node[len(node)-1].position + default: + return Position{0, 0} + } +} + +// GetDefault works like Get but with a default value +func (t *TomlTree) GetDefault(key string, def interface{}) interface{} { + val := t.Get(key) + if val == nil { + return def + } + return val +} + +// Set an element in the tree. +// Key is a dot-separated path (e.g. a.b.c). +// Creates all necessary intermediates trees, if needed. +func (t *TomlTree) Set(key string, value interface{}) { + t.SetPath(strings.Split(key, "."), value) +} + +// SetPath sets an element in the tree. +// Keys is an array of path elements (e.g. {"a","b","c"}). +// Creates all necessary intermediates trees, if needed. +func (t *TomlTree) SetPath(keys []string, value interface{}) { + subtree := t + for _, intermediateKey := range keys[:len(keys)-1] { + nextTree, exists := subtree.values[intermediateKey] + if !exists { + nextTree = newTomlTree() + subtree.values[intermediateKey] = nextTree // add new element here + } + switch node := nextTree.(type) { + case *TomlTree: + subtree = node + case []*TomlTree: + // go to most recent element + if len(node) == 0 { + // create element if it does not exist + subtree.values[intermediateKey] = append(node, newTomlTree()) + } + subtree = node[len(node)-1] + } + } + + var toInsert interface{} + + switch value.(type) { + case *TomlTree: + toInsert = value + case []*TomlTree: + toInsert = value + case *tomlValue: + toInsert = value + default: + toInsert = &tomlValue{value: value} + } + + subtree.values[keys[len(keys)-1]] = toInsert +} + +// createSubTree takes a tree and a key and create the necessary intermediate +// subtrees to create a subtree at that point. In-place. +// +// e.g. passing a.b.c will create (assuming tree is empty) tree[a], tree[a][b] +// and tree[a][b][c] +// +// Returns nil on success, error object on failure +func (t *TomlTree) createSubTree(keys []string, pos Position) error { + subtree := t + for _, intermediateKey := range keys { + nextTree, exists := subtree.values[intermediateKey] + if !exists { + tree := newTomlTree() + tree.position = pos + subtree.values[intermediateKey] = tree + nextTree = tree + } + + switch node := nextTree.(type) { + case []*TomlTree: + subtree = node[len(node)-1] + case *TomlTree: + subtree = node + default: + return fmt.Errorf("unknown type for path %s (%s): %T (%#v)", + strings.Join(keys, "."), intermediateKey, nextTree, nextTree) + } + } + return nil +} + +// Query compiles and executes a query on a tree and returns the query result. +func (t *TomlTree) Query(query string) (*QueryResult, error) { + q, err := CompileQuery(query) + if err != nil { + return nil, err + } + return q.Execute(t), nil +} + +// LoadReader creates a TomlTree from any io.Reader. +func LoadReader(reader io.Reader) (tree *TomlTree, err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = errors.New(r.(string)) + } + }() + tree = parseToml(lexToml(reader)) + return +} + +// Load creates a TomlTree from a string. +func Load(content string) (tree *TomlTree, err error) { + return LoadReader(strings.NewReader(content)) +} + +// LoadFile creates a TomlTree from a file. +func LoadFile(path string) (tree *TomlTree, err error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + return LoadReader(file) +} diff --git a/vendor/github.com/pelletier/go-toml/tomltree_create.go b/vendor/github.com/pelletier/go-toml/tomltree_create.go new file mode 100644 index 00000000..c6054f35 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/tomltree_create.go @@ -0,0 +1,135 @@ +package toml + +import ( + "fmt" + "reflect" + "time" +) + +// supported values: +// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32 + +var kindToTypeMapping = map[reflect.Kind]reflect.Type{ + reflect.Bool: reflect.TypeOf(true), + reflect.String: reflect.TypeOf(""), + reflect.Float32: reflect.TypeOf(float64(1)), + reflect.Float64: reflect.TypeOf(float64(1)), + reflect.Int: reflect.TypeOf(int64(1)), + reflect.Int8: reflect.TypeOf(int64(1)), + reflect.Int16: reflect.TypeOf(int64(1)), + reflect.Int32: reflect.TypeOf(int64(1)), + reflect.Int64: reflect.TypeOf(int64(1)), + reflect.Uint: reflect.TypeOf(uint64(1)), + reflect.Uint8: reflect.TypeOf(uint64(1)), + reflect.Uint16: reflect.TypeOf(uint64(1)), + reflect.Uint32: reflect.TypeOf(uint64(1)), + reflect.Uint64: reflect.TypeOf(uint64(1)), +} + +func simpleValueCoercion(object interface{}) (interface{}, error) { + switch original := object.(type) { + case string, bool, int64, uint64, float64, time.Time: + return original, nil + case int: + return int64(original), nil + case int8: + return int64(original), nil + case int16: + return int64(original), nil + case int32: + return int64(original), nil + case uint: + return uint64(original), nil + case uint8: + return uint64(original), nil + case uint16: + return uint64(original), nil + case uint32: + return uint64(original), nil + case float32: + return float64(original), nil + case fmt.Stringer: + return original.String(), nil + default: + return nil, fmt.Errorf("cannot convert type %T to TomlTree", object) + } +} + +func sliceToTree(object interface{}) (interface{}, error) { + // arrays are a bit tricky, since they can represent either a + // collection of simple values, which is represented by one + // *tomlValue, or an array of tables, which is represented by an + // array of *TomlTree. + + // holding the assumption that this function is called from toTree only when value.Kind() is Array or Slice + value := reflect.ValueOf(object) + insideType := value.Type().Elem() + length := value.Len() + if length > 0 { + insideType = reflect.ValueOf(value.Index(0).Interface()).Type() + } + if insideType.Kind() == reflect.Map { + // this is considered as an array of tables + tablesArray := make([]*TomlTree, 0, length) + for i := 0; i < length; i++ { + table := value.Index(i) + tree, err := toTree(table.Interface()) + if err != nil { + return nil, err + } + tablesArray = append(tablesArray, tree.(*TomlTree)) + } + return tablesArray, nil + } + + sliceType := kindToTypeMapping[insideType.Kind()] + if sliceType == nil { + sliceType = insideType + } + + arrayValue := reflect.MakeSlice(reflect.SliceOf(sliceType), 0, length) + + for i := 0; i < length; i++ { + val := value.Index(i).Interface() + simpleValue, err := simpleValueCoercion(val) + if err != nil { + return nil, err + } + arrayValue = reflect.Append(arrayValue, reflect.ValueOf(simpleValue)) + } + return &tomlValue{arrayValue.Interface(), Position{}}, nil +} + +func toTree(object interface{}) (interface{}, error) { + value := reflect.ValueOf(object) + + if value.Kind() == reflect.Map { + values := map[string]interface{}{} + keys := value.MapKeys() + for _, key := range keys { + if key.Kind() != reflect.String { + if _, ok := key.Interface().(string); !ok { + return nil, fmt.Errorf("map key needs to be a string, not %T (%v)", key.Interface(), key.Kind()) + } + } + + v := value.MapIndex(key) + newValue, err := toTree(v.Interface()) + if err != nil { + return nil, err + } + values[key.String()] = newValue + } + return &TomlTree{values, Position{}}, nil + } + + if value.Kind() == reflect.Array || value.Kind() == reflect.Slice { + return sliceToTree(object) + } + + simpleValue, err := simpleValueCoercion(object) + if err != nil { + return nil, err + } + return &tomlValue{simpleValue, Position{}}, nil +} diff --git a/vendor/github.com/pelletier/go-toml/tomltree_write.go b/vendor/github.com/pelletier/go-toml/tomltree_write.go new file mode 100644 index 00000000..6a7fa174 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/tomltree_write.go @@ -0,0 +1,217 @@ +package toml + +import ( + "bytes" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" + "reflect" +) + +// encodes a string to a TOML-compliant string value +func encodeTomlString(value string) string { + result := "" + for _, rr := range value { + switch rr { + case '\b': + result += "\\b" + case '\t': + result += "\\t" + case '\n': + result += "\\n" + case '\f': + result += "\\f" + case '\r': + result += "\\r" + case '"': + result += "\\\"" + case '\\': + result += "\\\\" + default: + intRr := uint16(rr) + if intRr < 0x001F { + result += fmt.Sprintf("\\u%0.4X", intRr) + } else { + result += string(rr) + } + } + } + return result +} + +func tomlValueStringRepresentation(v interface{}) (string, error) { + switch value := v.(type) { + case uint64: + return strconv.FormatUint(value, 10), nil + case int64: + return strconv.FormatInt(value, 10), nil + case float64: + return strconv.FormatFloat(value, 'f', -1, 32), nil + case string: + return "\"" + encodeTomlString(value) + "\"", nil + case []byte: + b, _ := v.([]byte) + return tomlValueStringRepresentation(string(b)) + case bool: + if value { + return "true", nil + } + return "false", nil + case time.Time: + return value.Format(time.RFC3339), nil + case nil: + return "", nil + } + + rv := reflect.ValueOf(v) + + if rv.Kind() == reflect.Slice { + values := []string{} + for i := 0; i < rv.Len(); i++ { + item := rv.Index(i).Interface() + itemRepr, err := tomlValueStringRepresentation(item) + if err != nil { + return "", err + } + values = append(values, itemRepr) + } + return "[" + strings.Join(values, ",") + "]", nil + } + return "", fmt.Errorf("unsupported value type %T: %v", v, v) +} + +func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (int64, error) { + simpleValuesKeys := make([]string, 0) + complexValuesKeys := make([]string, 0) + + for k := range t.values { + v := t.values[k] + switch v.(type) { + case *TomlTree, []*TomlTree: + complexValuesKeys = append(complexValuesKeys, k) + default: + simpleValuesKeys = append(simpleValuesKeys, k) + } + } + + sort.Strings(simpleValuesKeys) + sort.Strings(complexValuesKeys) + + for _, k := range simpleValuesKeys { + v, ok := t.values[k].(*tomlValue) + if !ok { + return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k]) + } + + repr, err := tomlValueStringRepresentation(v.value) + if err != nil { + return bytesCount, err + } + + kvRepr := fmt.Sprintf("%s%s = %s\n", indent, k, repr) + writtenBytesCount, err := w.Write([]byte(kvRepr)) + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + } + + for _, k := range complexValuesKeys { + v := t.values[k] + + combinedKey := k + if keyspace != "" { + combinedKey = keyspace + "." + combinedKey + } + + switch node := v.(type) { + // node has to be of those two types given how keys are sorted above + case *TomlTree: + tableName := fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) + writtenBytesCount, err := w.Write([]byte(tableName)) + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount) + if err != nil { + return bytesCount, err + } + case []*TomlTree: + for _, subTree := range node { + if len(subTree.values) > 0 { + tableArrayName := fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey) + writtenBytesCount, err := w.Write([]byte(tableArrayName)) + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } + + bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount) + if err != nil { + return bytesCount, err + } + } + } + } + } + + return bytesCount, nil +} + +// WriteTo encode the TomlTree as Toml and writes it to the writer w. +// Returns the number of bytes written in case of success, or an error if anything happened. +func (t *TomlTree) WriteTo(w io.Writer) (int64, error) { + return t.writeTo(w, "", "", 0) +} + +// ToTomlString generates a human-readable representation of the current tree. +// Output spans multiple lines, and is suitable for ingest by a TOML parser. +// If the conversion cannot be performed, ToString returns a non-nil error. +func (t *TomlTree) ToTomlString() (string, error) { + var buf bytes.Buffer + _, err := t.WriteTo(&buf) + if err != nil { + return "", err + } + return buf.String(), nil +} + +// String generates a human-readable representation of the current tree. +// Alias of ToString. Present to implement the fmt.Stringer interface. +func (t *TomlTree) String() string { + result, _ := t.ToTomlString() + return result +} + +// ToMap recursively generates a representation of the tree using Go built-in structures. +// The following types are used: +// * uint64 +// * int64 +// * bool +// * string +// * time.Time +// * map[string]interface{} (where interface{} is any of this list) +// * []interface{} (where interface{} is any of this list) +func (t *TomlTree) ToMap() map[string]interface{} { + result := map[string]interface{}{} + + for k, v := range t.values { + switch node := v.(type) { + case []*TomlTree: + var array []interface{} + for _, item := range node { + array = append(array, item.ToMap()) + } + result[k] = array + case *TomlTree: + result[k] = node.ToMap() + case *tomlValue: + result[k] = node.value + } + } + return result +} diff --git a/vendor/github.com/pmezard/go-difflib/LICENSE b/vendor/github.com/pmezard/go-difflib/LICENSE new file mode 100644 index 00000000..c67dad61 --- /dev/null +++ b/vendor/github.com/pmezard/go-difflib/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013, Patrick Mezard +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + The names of its contributors may not be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/pmezard/go-difflib/difflib/difflib_test.go b/vendor/github.com/pmezard/go-difflib/difflib/difflib_test.go deleted file mode 100644 index d7251196..00000000 --- a/vendor/github.com/pmezard/go-difflib/difflib/difflib_test.go +++ /dev/null @@ -1,426 +0,0 @@ -package difflib - -import ( - "bytes" - "fmt" - "math" - "reflect" - "strings" - "testing" -) - -func assertAlmostEqual(t *testing.T, a, b float64, places int) { - if math.Abs(a-b) > math.Pow10(-places) { - t.Errorf("%.7f != %.7f", a, b) - } -} - -func assertEqual(t *testing.T, a, b interface{}) { - if !reflect.DeepEqual(a, b) { - t.Errorf("%v != %v", a, b) - } -} - -func splitChars(s string) []string { - chars := make([]string, 0, len(s)) - // Assume ASCII inputs - for i := 0; i != len(s); i++ { - chars = append(chars, string(s[i])) - } - return chars -} - -func TestSequenceMatcherRatio(t *testing.T) { - s := NewMatcher(splitChars("abcd"), splitChars("bcde")) - assertEqual(t, s.Ratio(), 0.75) - assertEqual(t, s.QuickRatio(), 0.75) - assertEqual(t, s.RealQuickRatio(), 1.0) -} - -func TestGetOptCodes(t *testing.T) { - a := "qabxcd" - b := "abycdf" - s := NewMatcher(splitChars(a), splitChars(b)) - w := &bytes.Buffer{} - for _, op := range s.GetOpCodes() { - fmt.Fprintf(w, "%s a[%d:%d], (%s) b[%d:%d] (%s)\n", string(op.Tag), - op.I1, op.I2, a[op.I1:op.I2], op.J1, op.J2, b[op.J1:op.J2]) - } - result := string(w.Bytes()) - expected := `d a[0:1], (q) b[0:0] () -e a[1:3], (ab) b[0:2] (ab) -r a[3:4], (x) b[2:3] (y) -e a[4:6], (cd) b[3:5] (cd) -i a[6:6], () b[5:6] (f) -` - if expected != result { - t.Errorf("unexpected op codes: \n%s", result) - } -} - -func TestGroupedOpCodes(t *testing.T) { - a := []string{} - for i := 0; i != 39; i++ { - a = append(a, fmt.Sprintf("%02d", i)) - } - b := []string{} - b = append(b, a[:8]...) - b = append(b, " i") - b = append(b, a[8:19]...) - b = append(b, " x") - b = append(b, a[20:22]...) - b = append(b, a[27:34]...) - b = append(b, " y") - b = append(b, a[35:]...) - s := NewMatcher(a, b) - w := &bytes.Buffer{} - for _, g := range s.GetGroupedOpCodes(-1) { - fmt.Fprintf(w, "group\n") - for _, op := range g { - fmt.Fprintf(w, " %s, %d, %d, %d, %d\n", string(op.Tag), - op.I1, op.I2, op.J1, op.J2) - } - } - result := string(w.Bytes()) - expected := `group - e, 5, 8, 5, 8 - i, 8, 8, 8, 9 - e, 8, 11, 9, 12 -group - e, 16, 19, 17, 20 - r, 19, 20, 20, 21 - e, 20, 22, 21, 23 - d, 22, 27, 23, 23 - e, 27, 30, 23, 26 -group - e, 31, 34, 27, 30 - r, 34, 35, 30, 31 - e, 35, 38, 31, 34 -` - if expected != result { - t.Errorf("unexpected op codes: \n%s", result) - } -} - -func ExampleGetUnifiedDiffCode() { - a := `one -two -three -four -fmt.Printf("%s,%T",a,b)` - b := `zero -one -three -four` - diff := UnifiedDiff{ - A: SplitLines(a), - B: SplitLines(b), - FromFile: "Original", - FromDate: "2005-01-26 23:30:50", - ToFile: "Current", - ToDate: "2010-04-02 10:20:52", - Context: 3, - } - result, _ := GetUnifiedDiffString(diff) - fmt.Println(strings.Replace(result, "\t", " ", -1)) - // Output: - // --- Original 2005-01-26 23:30:50 - // +++ Current 2010-04-02 10:20:52 - // @@ -1,5 +1,4 @@ - // +zero - // one - // -two - // three - // four - // -fmt.Printf("%s,%T",a,b) -} - -func ExampleGetContextDiffCode() { - a := `one -two -three -four -fmt.Printf("%s,%T",a,b)` - b := `zero -one -tree -four` - diff := ContextDiff{ - A: SplitLines(a), - B: SplitLines(b), - FromFile: "Original", - ToFile: "Current", - Context: 3, - Eol: "\n", - } - result, _ := GetContextDiffString(diff) - fmt.Print(strings.Replace(result, "\t", " ", -1)) - // Output: - // *** Original - // --- Current - // *************** - // *** 1,5 **** - // one - // ! two - // ! three - // four - // - fmt.Printf("%s,%T",a,b) - // --- 1,4 ---- - // + zero - // one - // ! tree - // four -} - -func ExampleGetContextDiffString() { - a := `one -two -three -four` - b := `zero -one -tree -four` - diff := ContextDiff{ - A: SplitLines(a), - B: SplitLines(b), - FromFile: "Original", - ToFile: "Current", - Context: 3, - Eol: "\n", - } - result, _ := GetContextDiffString(diff) - fmt.Printf(strings.Replace(result, "\t", " ", -1)) - // Output: - // *** Original - // --- Current - // *************** - // *** 1,4 **** - // one - // ! two - // ! three - // four - // --- 1,4 ---- - // + zero - // one - // ! tree - // four -} - -func rep(s string, count int) string { - return strings.Repeat(s, count) -} - -func TestWithAsciiOneInsert(t *testing.T) { - sm := NewMatcher(splitChars(rep("b", 100)), - splitChars("a"+rep("b", 100))) - assertAlmostEqual(t, sm.Ratio(), 0.995, 3) - assertEqual(t, sm.GetOpCodes(), - []OpCode{{'i', 0, 0, 0, 1}, {'e', 0, 100, 1, 101}}) - assertEqual(t, len(sm.bPopular), 0) - - sm = NewMatcher(splitChars(rep("b", 100)), - splitChars(rep("b", 50)+"a"+rep("b", 50))) - assertAlmostEqual(t, sm.Ratio(), 0.995, 3) - assertEqual(t, sm.GetOpCodes(), - []OpCode{{'e', 0, 50, 0, 50}, {'i', 50, 50, 50, 51}, {'e', 50, 100, 51, 101}}) - assertEqual(t, len(sm.bPopular), 0) -} - -func TestWithAsciiOnDelete(t *testing.T) { - sm := NewMatcher(splitChars(rep("a", 40)+"c"+rep("b", 40)), - splitChars(rep("a", 40)+rep("b", 40))) - assertAlmostEqual(t, sm.Ratio(), 0.994, 3) - assertEqual(t, sm.GetOpCodes(), - []OpCode{{'e', 0, 40, 0, 40}, {'d', 40, 41, 40, 40}, {'e', 41, 81, 40, 80}}) -} - -func TestWithAsciiBJunk(t *testing.T) { - isJunk := func(s string) bool { - return s == " " - } - sm := NewMatcherWithJunk(splitChars(rep("a", 40)+rep("b", 40)), - splitChars(rep("a", 44)+rep("b", 40)), true, isJunk) - assertEqual(t, sm.bJunk, map[string]struct{}{}) - - sm = NewMatcherWithJunk(splitChars(rep("a", 40)+rep("b", 40)), - splitChars(rep("a", 44)+rep("b", 40)+rep(" ", 20)), false, isJunk) - assertEqual(t, sm.bJunk, map[string]struct{}{" ": struct{}{}}) - - isJunk = func(s string) bool { - return s == " " || s == "b" - } - sm = NewMatcherWithJunk(splitChars(rep("a", 40)+rep("b", 40)), - splitChars(rep("a", 44)+rep("b", 40)+rep(" ", 20)), false, isJunk) - assertEqual(t, sm.bJunk, map[string]struct{}{" ": struct{}{}, "b": struct{}{}}) -} - -func TestSFBugsRatioForNullSeqn(t *testing.T) { - sm := NewMatcher(nil, nil) - assertEqual(t, sm.Ratio(), 1.0) - assertEqual(t, sm.QuickRatio(), 1.0) - assertEqual(t, sm.RealQuickRatio(), 1.0) -} - -func TestSFBugsComparingEmptyLists(t *testing.T) { - groups := NewMatcher(nil, nil).GetGroupedOpCodes(-1) - assertEqual(t, len(groups), 0) - diff := UnifiedDiff{ - FromFile: "Original", - ToFile: "Current", - Context: 3, - } - result, err := GetUnifiedDiffString(diff) - assertEqual(t, err, nil) - assertEqual(t, result, "") -} - -func TestOutputFormatRangeFormatUnified(t *testing.T) { - // Per the diff spec at http://www.unix.org/single_unix_specification/ - // - // Each field shall be of the form: - // %1d", if the range contains exactly one line, - // and: - // "%1d,%1d", , otherwise. - // If a range is empty, its beginning line number shall be the number of - // the line just before the range, or 0 if the empty range starts the file. - fm := formatRangeUnified - assertEqual(t, fm(3, 3), "3,0") - assertEqual(t, fm(3, 4), "4") - assertEqual(t, fm(3, 5), "4,2") - assertEqual(t, fm(3, 6), "4,3") - assertEqual(t, fm(0, 0), "0,0") -} - -func TestOutputFormatRangeFormatContext(t *testing.T) { - // Per the diff spec at http://www.unix.org/single_unix_specification/ - // - // The range of lines in file1 shall be written in the following format - // if the range contains two or more lines: - // "*** %d,%d ****\n", , - // and the following format otherwise: - // "*** %d ****\n", - // The ending line number of an empty range shall be the number of the preceding line, - // or 0 if the range is at the start of the file. - // - // Next, the range of lines in file2 shall be written in the following format - // if the range contains two or more lines: - // "--- %d,%d ----\n", , - // and the following format otherwise: - // "--- %d ----\n", - fm := formatRangeContext - assertEqual(t, fm(3, 3), "3") - assertEqual(t, fm(3, 4), "4") - assertEqual(t, fm(3, 5), "4,5") - assertEqual(t, fm(3, 6), "4,6") - assertEqual(t, fm(0, 0), "0") -} - -func TestOutputFormatTabDelimiter(t *testing.T) { - diff := UnifiedDiff{ - A: splitChars("one"), - B: splitChars("two"), - FromFile: "Original", - FromDate: "2005-01-26 23:30:50", - ToFile: "Current", - ToDate: "2010-04-12 10:20:52", - Eol: "\n", - } - ud, err := GetUnifiedDiffString(diff) - assertEqual(t, err, nil) - assertEqual(t, SplitLines(ud)[:2], []string{ - "--- Original\t2005-01-26 23:30:50\n", - "+++ Current\t2010-04-12 10:20:52\n", - }) - cd, err := GetContextDiffString(ContextDiff(diff)) - assertEqual(t, err, nil) - assertEqual(t, SplitLines(cd)[:2], []string{ - "*** Original\t2005-01-26 23:30:50\n", - "--- Current\t2010-04-12 10:20:52\n", - }) -} - -func TestOutputFormatNoTrailingTabOnEmptyFiledate(t *testing.T) { - diff := UnifiedDiff{ - A: splitChars("one"), - B: splitChars("two"), - FromFile: "Original", - ToFile: "Current", - Eol: "\n", - } - ud, err := GetUnifiedDiffString(diff) - assertEqual(t, err, nil) - assertEqual(t, SplitLines(ud)[:2], []string{"--- Original\n", "+++ Current\n"}) - - cd, err := GetContextDiffString(ContextDiff(diff)) - assertEqual(t, err, nil) - assertEqual(t, SplitLines(cd)[:2], []string{"*** Original\n", "--- Current\n"}) -} - -func TestOmitFilenames(t *testing.T) { - diff := UnifiedDiff{ - A: SplitLines("o\nn\ne\n"), - B: SplitLines("t\nw\no\n"), - Eol: "\n", - } - ud, err := GetUnifiedDiffString(diff) - assertEqual(t, err, nil) - assertEqual(t, SplitLines(ud), []string{ - "@@ -0,0 +1,2 @@\n", - "+t\n", - "+w\n", - "@@ -2,2 +3,0 @@\n", - "-n\n", - "-e\n", - "\n", - }) - - cd, err := GetContextDiffString(ContextDiff(diff)) - assertEqual(t, err, nil) - assertEqual(t, SplitLines(cd), []string{ - "***************\n", - "*** 0 ****\n", - "--- 1,2 ----\n", - "+ t\n", - "+ w\n", - "***************\n", - "*** 2,3 ****\n", - "- n\n", - "- e\n", - "--- 3 ----\n", - "\n", - }) -} - -func TestSplitLines(t *testing.T) { - allTests := []struct { - input string - want []string - }{ - {"foo", []string{"foo\n"}}, - {"foo\nbar", []string{"foo\n", "bar\n"}}, - {"foo\nbar\n", []string{"foo\n", "bar\n", "\n"}}, - } - for _, test := range allTests { - assertEqual(t, SplitLines(test.input), test.want) - } -} - -func benchmarkSplitLines(b *testing.B, count int) { - str := strings.Repeat("foo\n", count) - - b.ResetTimer() - - n := 0 - for i := 0; i < b.N; i++ { - n += len(SplitLines(str)) - } -} - -func BenchmarkSplitLines100(b *testing.B) { - benchmarkSplitLines(b, 100) -} - -func BenchmarkSplitLines10000(b *testing.B) { - benchmarkSplitLines(b, 10000) -} diff --git a/vendor/github.com/russross/blackfriday/LICENSE.txt b/vendor/github.com/russross/blackfriday/LICENSE.txt deleted file mode 100644 index 2885af36..00000000 --- a/vendor/github.com/russross/blackfriday/LICENSE.txt +++ /dev/null @@ -1,29 +0,0 @@ -Blackfriday is distributed under the Simplified BSD License: - -> Copyright © 2011 Russ Ross -> All rights reserved. -> -> Redistribution and use in source and binary forms, with or without -> modification, are permitted provided that the following conditions -> are met: -> -> 1. Redistributions of source code must retain the above copyright -> notice, this list of conditions and the following disclaimer. -> -> 2. Redistributions in binary form must reproduce the above -> copyright notice, this list of conditions and the following -> disclaimer in the documentation and/or other materials provided with -> the distribution. -> -> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -> "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -> FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -> COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -> INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -> BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -> LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -> CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -> LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -> ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -> POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/russross/blackfriday/README.md b/vendor/github.com/russross/blackfriday/README.md deleted file mode 100644 index dd8eb161..00000000 --- a/vendor/github.com/russross/blackfriday/README.md +++ /dev/null @@ -1,267 +0,0 @@ -Blackfriday [![Build Status](https://travis-ci.org/russross/blackfriday.svg?branch=master)](https://travis-ci.org/russross/blackfriday) -=========== - -Blackfriday is a [Markdown][1] processor implemented in [Go][2]. It -is paranoid about its input (so you can safely feed it user-supplied -data), it is fast, it supports common extensions (tables, smart -punctuation substitutions, etc.), and it is safe for all utf-8 -(unicode) input. - -HTML output is currently supported, along with Smartypants -extensions. An experimental LaTeX output engine is also included. - -It started as a translation from C of [Sundown][3]. - - -Installation ------------- - -Blackfriday is compatible with Go 1. If you are using an older -release of Go, consider using v1.1 of blackfriday, which was based -on the last stable release of Go prior to Go 1. You can find it as a -tagged commit on github. - -With Go 1 and git installed: - - go get github.com/russross/blackfriday - -will download, compile, and install the package into your `$GOPATH` -directory hierarchy. Alternatively, you can achieve the same if you -import it into a project: - - import "github.com/russross/blackfriday" - -and `go get` without parameters. - -Usage ------ - -For basic usage, it is as simple as getting your input into a byte -slice and calling: - - output := blackfriday.MarkdownBasic(input) - -This renders it with no extensions enabled. To get a more useful -feature set, use this instead: - - output := blackfriday.MarkdownCommon(input) - -### Sanitize untrusted content - -Blackfriday itself does nothing to protect against malicious content. If you are -dealing with user-supplied markdown, we recommend running blackfriday's output -through HTML sanitizer such as -[Bluemonday](https://github.com/microcosm-cc/bluemonday). - -Here's an example of simple usage of blackfriday together with bluemonday: - -``` go -import ( - "github.com/microcosm-cc/bluemonday" - "github.com/russross/blackfriday" -) - -// ... -unsafe := blackfriday.MarkdownCommon(input) -html := bluemonday.UGCPolicy().SanitizeBytes(unsafe) -``` - -### Custom options - -If you want to customize the set of options, first get a renderer -(currently either the HTML or LaTeX output engines), then use it to -call the more general `Markdown` function. For examples, see the -implementations of `MarkdownBasic` and `MarkdownCommon` in -`markdown.go`. - -You can also check out `blackfriday-tool` for a more complete example -of how to use it. Download and install it using: - - go get github.com/russross/blackfriday-tool - -This is a simple command-line tool that allows you to process a -markdown file using a standalone program. You can also browse the -source directly on github if you are just looking for some example -code: - -* - -Note that if you have not already done so, installing -`blackfriday-tool` will be sufficient to download and install -blackfriday in addition to the tool itself. The tool binary will be -installed in `$GOPATH/bin`. This is a statically-linked binary that -can be copied to wherever you need it without worrying about -dependencies and library versions. - - -Features --------- - -All features of Sundown are supported, including: - -* **Compatibility**. The Markdown v1.0.3 test suite passes with - the `--tidy` option. Without `--tidy`, the differences are - mostly in whitespace and entity escaping, where blackfriday is - more consistent and cleaner. - -* **Common extensions**, including table support, fenced code - blocks, autolinks, strikethroughs, non-strict emphasis, etc. - -* **Safety**. Blackfriday is paranoid when parsing, making it safe - to feed untrusted user input without fear of bad things - happening. The test suite stress tests this and there are no - known inputs that make it crash. If you find one, please let me - know and send me the input that does it. - - NOTE: "safety" in this context means *runtime safety only*. In order to - protect yourself agains JavaScript injection in untrusted content, see - [this example](https://github.com/russross/blackfriday#sanitize-untrusted-content). - -* **Fast processing**. It is fast enough to render on-demand in - most web applications without having to cache the output. - -* **Thread safety**. You can run multiple parsers in different - goroutines without ill effect. There is no dependence on global - shared state. - -* **Minimal dependencies**. Blackfriday only depends on standard - library packages in Go. The source code is pretty - self-contained, so it is easy to add to any project, including - Google App Engine projects. - -* **Standards compliant**. Output successfully validates using the - W3C validation tool for HTML 4.01 and XHTML 1.0 Transitional. - - -Extensions ----------- - -In addition to the standard markdown syntax, this package -implements the following extensions: - -* **Intra-word emphasis supression**. The `_` character is - commonly used inside words when discussing code, so having - markdown interpret it as an emphasis command is usually the - wrong thing. Blackfriday lets you treat all emphasis markers as - normal characters when they occur inside a word. - -* **Tables**. Tables can be created by drawing them in the input - using a simple syntax: - - ``` - Name | Age - --------|------ - Bob | 27 - Alice | 23 - ``` - -* **Fenced code blocks**. In addition to the normal 4-space - indentation to mark code blocks, you can explicitly mark them - and supply a language (to make syntax highlighting simple). Just - mark it like this: - - ``` go - func getTrue() bool { - return true - } - ``` - - You can use 3 or more backticks to mark the beginning of the - block, and the same number to mark the end of the block. - -* **Definition lists**. A simple definition list is made of a single-line - term followed by a colon and the definition for that term. - - Cat - : Fluffy animal everyone likes - - Internet - : Vector of transmission for pictures of cats - - Terms must be separated from the previous definition by a blank line. - -* **Footnotes**. A marker in the text that will become a superscript number; - a footnote definition that will be placed in a list of footnotes at the - end of the document. A footnote looks like this: - - This is a footnote.[^1] - - [^1]: the footnote text. - -* **Autolinking**. Blackfriday can find URLs that have not been - explicitly marked as links and turn them into links. - -* **Strikethrough**. Use two tildes (`~~`) to mark text that - should be crossed out. - -* **Hard line breaks**. With this extension enabled (it is off by - default in the `MarkdownBasic` and `MarkdownCommon` convenience - functions), newlines in the input translate into line breaks in - the output. - -* **Smart quotes**. Smartypants-style punctuation substitution is - supported, turning normal double- and single-quote marks into - curly quotes, etc. - -* **LaTeX-style dash parsing** is an additional option, where `--` - is translated into `–`, and `---` is translated into - `—`. This differs from most smartypants processors, which - turn a single hyphen into an ndash and a double hyphen into an - mdash. - -* **Smart fractions**, where anything that looks like a fraction - is translated into suitable HTML (instead of just a few special - cases like most smartypant processors). For example, `4/5` - becomes `4⁄5`, which renders as - 4⁄5. - - -Other renderers ---------------- - -Blackfriday is structured to allow alternative rendering engines. Here -are a few of note: - -* [github_flavored_markdown](https://godoc.org/github.com/shurcooL/github_flavored_markdown): - provides a GitHub Flavored Markdown renderer with fenced code block - highlighting, clickable header anchor links. - - It's not customizable, and its goal is to produce HTML output - equivalent to the [GitHub Markdown API endpoint](https://developer.github.com/v3/markdown/#render-a-markdown-document-in-raw-mode), - except the rendering is performed locally. - -* [markdownfmt](https://github.com/shurcooL/markdownfmt): like gofmt, - but for markdown. - -* LaTeX output: renders output as LaTeX. This is currently part of the - main Blackfriday repository, but may be split into its own project - in the future. If you are interested in owning and maintaining the - LaTeX output component, please be in touch. - - It renders some basic documents, but is only experimental at this - point. In particular, it does not do any inline escaping, so input - that happens to look like LaTeX code will be passed through without - modification. - -* [Md2Vim](https://github.com/FooSoft/md2vim): transforms markdown files into vimdoc format. - - -Todo ----- - -* More unit testing -* Improve unicode support. It does not understand all unicode - rules (about what constitutes a letter, a punctuation symbol, - etc.), so it may fail to detect word boundaries correctly in - some instances. It is safe on all utf-8 input. - - -License -------- - -[Blackfriday is distributed under the Simplified BSD License](LICENSE.txt) - - - [1]: http://daringfireball.net/projects/markdown/ "Markdown" - [2]: http://golang.org/ "Go Language" - [3]: https://github.com/vmg/sundown "Sundown" diff --git a/vendor/github.com/russross/blackfriday/block.go b/vendor/github.com/russross/blackfriday/block.go deleted file mode 100644 index 740ad462..00000000 --- a/vendor/github.com/russross/blackfriday/block.go +++ /dev/null @@ -1,1412 +0,0 @@ -// -// Blackfriday Markdown Processor -// Available at http://github.com/russross/blackfriday -// -// Copyright © 2011 Russ Ross . -// Distributed under the Simplified BSD License. -// See README.md for details. -// - -// -// Functions to parse block-level elements. -// - -package blackfriday - -import ( - "bytes" - - "github.com/shurcooL/sanitized_anchor_name" -) - -// Parse block-level data. -// Note: this function and many that it calls assume that -// the input buffer ends with a newline. -func (p *parser) block(out *bytes.Buffer, data []byte) { - if len(data) == 0 || data[len(data)-1] != '\n' { - panic("block input is missing terminating newline") - } - - // this is called recursively: enforce a maximum depth - if p.nesting >= p.maxNesting { - return - } - p.nesting++ - - // parse out one block-level construct at a time - for len(data) > 0 { - // prefixed header: - // - // # Header 1 - // ## Header 2 - // ... - // ###### Header 6 - if p.isPrefixHeader(data) { - data = data[p.prefixHeader(out, data):] - continue - } - - // block of preformatted HTML: - // - //
- // ... - //
- if data[0] == '<' { - if i := p.html(out, data, true); i > 0 { - data = data[i:] - continue - } - } - - // title block - // - // % stuff - // % more stuff - // % even more stuff - if p.flags&EXTENSION_TITLEBLOCK != 0 { - if data[0] == '%' { - if i := p.titleBlock(out, data, true); i > 0 { - data = data[i:] - continue - } - } - } - - // blank lines. note: returns the # of bytes to skip - if i := p.isEmpty(data); i > 0 { - data = data[i:] - continue - } - - // indented code block: - // - // func max(a, b int) int { - // if a > b { - // return a - // } - // return b - // } - if p.codePrefix(data) > 0 { - data = data[p.code(out, data):] - continue - } - - // fenced code block: - // - // ``` go - // func fact(n int) int { - // if n <= 1 { - // return n - // } - // return n * fact(n-1) - // } - // ``` - if p.flags&EXTENSION_FENCED_CODE != 0 { - if i := p.fencedCode(out, data, true); i > 0 { - data = data[i:] - continue - } - } - - // horizontal rule: - // - // ------ - // or - // ****** - // or - // ______ - if p.isHRule(data) { - p.r.HRule(out) - var i int - for i = 0; data[i] != '\n'; i++ { - } - data = data[i:] - continue - } - - // block quote: - // - // > A big quote I found somewhere - // > on the web - if p.quotePrefix(data) > 0 { - data = data[p.quote(out, data):] - continue - } - - // table: - // - // Name | Age | Phone - // ------|-----|--------- - // Bob | 31 | 555-1234 - // Alice | 27 | 555-4321 - if p.flags&EXTENSION_TABLES != 0 { - if i := p.table(out, data); i > 0 { - data = data[i:] - continue - } - } - - // an itemized/unordered list: - // - // * Item 1 - // * Item 2 - // - // also works with + or - - if p.uliPrefix(data) > 0 { - data = data[p.list(out, data, 0):] - continue - } - - // a numbered/ordered list: - // - // 1. Item 1 - // 2. Item 2 - if p.oliPrefix(data) > 0 { - data = data[p.list(out, data, LIST_TYPE_ORDERED):] - continue - } - - // definition lists: - // - // Term 1 - // : Definition a - // : Definition b - // - // Term 2 - // : Definition c - if p.flags&EXTENSION_DEFINITION_LISTS != 0 { - if p.dliPrefix(data) > 0 { - data = data[p.list(out, data, LIST_TYPE_DEFINITION):] - continue - } - } - - // anything else must look like a normal paragraph - // note: this finds underlined headers, too - data = data[p.paragraph(out, data):] - } - - p.nesting-- -} - -func (p *parser) isPrefixHeader(data []byte) bool { - if data[0] != '#' { - return false - } - - if p.flags&EXTENSION_SPACE_HEADERS != 0 { - level := 0 - for level < 6 && data[level] == '#' { - level++ - } - if data[level] != ' ' { - return false - } - } - return true -} - -func (p *parser) prefixHeader(out *bytes.Buffer, data []byte) int { - level := 0 - for level < 6 && data[level] == '#' { - level++ - } - i := skipChar(data, level, ' ') - end := skipUntilChar(data, i, '\n') - skip := end - id := "" - if p.flags&EXTENSION_HEADER_IDS != 0 { - j, k := 0, 0 - // find start/end of header id - for j = i; j < end-1 && (data[j] != '{' || data[j+1] != '#'); j++ { - } - for k = j + 1; k < end && data[k] != '}'; k++ { - } - // extract header id iff found - if j < end && k < end { - id = string(data[j+2 : k]) - end = j - skip = k + 1 - for end > 0 && data[end-1] == ' ' { - end-- - } - } - } - for end > 0 && data[end-1] == '#' { - if isBackslashEscaped(data, end-1) { - break - } - end-- - } - for end > 0 && data[end-1] == ' ' { - end-- - } - if end > i { - if id == "" && p.flags&EXTENSION_AUTO_HEADER_IDS != 0 { - id = sanitized_anchor_name.Create(string(data[i:end])) - } - work := func() bool { - p.inline(out, data[i:end]) - return true - } - p.r.Header(out, work, level, id) - } - return skip -} - -func (p *parser) isUnderlinedHeader(data []byte) int { - // test of level 1 header - if data[0] == '=' { - i := skipChar(data, 1, '=') - i = skipChar(data, i, ' ') - if data[i] == '\n' { - return 1 - } else { - return 0 - } - } - - // test of level 2 header - if data[0] == '-' { - i := skipChar(data, 1, '-') - i = skipChar(data, i, ' ') - if data[i] == '\n' { - return 2 - } else { - return 0 - } - } - - return 0 -} - -func (p *parser) titleBlock(out *bytes.Buffer, data []byte, doRender bool) int { - if data[0] != '%' { - return 0 - } - splitData := bytes.Split(data, []byte("\n")) - var i int - for idx, b := range splitData { - if !bytes.HasPrefix(b, []byte("%")) { - i = idx // - 1 - break - } - } - - data = bytes.Join(splitData[0:i], []byte("\n")) - p.r.TitleBlock(out, data) - - return len(data) -} - -func (p *parser) html(out *bytes.Buffer, data []byte, doRender bool) int { - var i, j int - - // identify the opening tag - if data[0] != '<' { - return 0 - } - curtag, tagfound := p.htmlFindTag(data[1:]) - - // handle special cases - if !tagfound { - // check for an HTML comment - if size := p.htmlComment(out, data, doRender); size > 0 { - return size - } - - // check for an
tag - if size := p.htmlHr(out, data, doRender); size > 0 { - return size - } - - // check for HTML CDATA - if size := p.htmlCDATA(out, data, doRender); size > 0 { - return size - } - - // no special case recognized - return 0 - } - - // look for an unindented matching closing tag - // followed by a blank line - found := false - /* - closetag := []byte("\n") - j = len(curtag) + 1 - for !found { - // scan for a closing tag at the beginning of a line - if skip := bytes.Index(data[j:], closetag); skip >= 0 { - j += skip + len(closetag) - } else { - break - } - - // see if it is the only thing on the line - if skip := p.isEmpty(data[j:]); skip > 0 { - // see if it is followed by a blank line/eof - j += skip - if j >= len(data) { - found = true - i = j - } else { - if skip := p.isEmpty(data[j:]); skip > 0 { - j += skip - found = true - i = j - } - } - } - } - */ - - // if not found, try a second pass looking for indented match - // but not if tag is "ins" or "del" (following original Markdown.pl) - if !found && curtag != "ins" && curtag != "del" { - i = 1 - for i < len(data) { - i++ - for i < len(data) && !(data[i-1] == '<' && data[i] == '/') { - i++ - } - - if i+2+len(curtag) >= len(data) { - break - } - - j = p.htmlFindEnd(curtag, data[i-1:]) - - if j > 0 { - i += j - 1 - found = true - break - } - } - } - - if !found { - return 0 - } - - // the end of the block has been found - if doRender { - // trim newlines - end := i - for end > 0 && data[end-1] == '\n' { - end-- - } - p.r.BlockHtml(out, data[:end]) - } - - return i -} - -func (p *parser) renderHTMLBlock(out *bytes.Buffer, data []byte, start int, doRender bool) int { - // html block needs to end with a blank line - if i := p.isEmpty(data[start:]); i > 0 { - size := start + i - if doRender { - // trim trailing newlines - end := size - for end > 0 && data[end-1] == '\n' { - end-- - } - p.r.BlockHtml(out, data[:end]) - } - return size - } - return 0 -} - -// HTML comment, lax form -func (p *parser) htmlComment(out *bytes.Buffer, data []byte, doRender bool) int { - i := p.inlineHTMLComment(out, data) - return p.renderHTMLBlock(out, data, i, doRender) -} - -// HTML CDATA section -func (p *parser) htmlCDATA(out *bytes.Buffer, data []byte, doRender bool) int { - const cdataTag = "') { - i++ - } - i++ - // no end-of-comment marker - if i >= len(data) { - return 0 - } - return p.renderHTMLBlock(out, data, i, doRender) -} - -// HR, which is the only self-closing block tag considered -func (p *parser) htmlHr(out *bytes.Buffer, data []byte, doRender bool) int { - if data[0] != '<' || (data[1] != 'h' && data[1] != 'H') || (data[2] != 'r' && data[2] != 'R') { - return 0 - } - if data[3] != ' ' && data[3] != '/' && data[3] != '>' { - // not an
tag after all; at least not a valid one - return 0 - } - - i := 3 - for data[i] != '>' && data[i] != '\n' { - i++ - } - - if data[i] == '>' { - return p.renderHTMLBlock(out, data, i+1, doRender) - } - - return 0 -} - -func (p *parser) htmlFindTag(data []byte) (string, bool) { - i := 0 - for isalnum(data[i]) { - i++ - } - key := string(data[:i]) - if _, ok := blockTags[key]; ok { - return key, true - } - return "", false -} - -func (p *parser) htmlFindEnd(tag string, data []byte) int { - // assume data[0] == '<' && data[1] == '/' already tested - - // check if tag is a match - closetag := []byte("") - if !bytes.HasPrefix(data, closetag) { - return 0 - } - i := len(closetag) - - // check that the rest of the line is blank - skip := 0 - if skip = p.isEmpty(data[i:]); skip == 0 { - return 0 - } - i += skip - skip = 0 - - if i >= len(data) { - return i - } - - if p.flags&EXTENSION_LAX_HTML_BLOCKS != 0 { - return i - } - if skip = p.isEmpty(data[i:]); skip == 0 { - // following line must be blank - return 0 - } - - return i + skip -} - -func (p *parser) isEmpty(data []byte) int { - // it is okay to call isEmpty on an empty buffer - if len(data) == 0 { - return 0 - } - - var i int - for i = 0; i < len(data) && data[i] != '\n'; i++ { - if data[i] != ' ' && data[i] != '\t' { - return 0 - } - } - return i + 1 -} - -func (p *parser) isHRule(data []byte) bool { - i := 0 - - // skip up to three spaces - for i < 3 && data[i] == ' ' { - i++ - } - - // look at the hrule char - if data[i] != '*' && data[i] != '-' && data[i] != '_' { - return false - } - c := data[i] - - // the whole line must be the char or whitespace - n := 0 - for data[i] != '\n' { - switch { - case data[i] == c: - n++ - case data[i] != ' ': - return false - } - i++ - } - - return n >= 3 -} - -func (p *parser) isFencedCode(data []byte, syntax **string, oldmarker string) (skip int, marker string) { - i, size := 0, 0 - skip = 0 - - // skip up to three spaces - for i < len(data) && i < 3 && data[i] == ' ' { - i++ - } - if i >= len(data) { - return - } - - // check for the marker characters: ~ or ` - if data[i] != '~' && data[i] != '`' { - return - } - - c := data[i] - - // the whole line must be the same char or whitespace - for i < len(data) && data[i] == c { - size++ - i++ - } - - if i >= len(data) { - return - } - - // the marker char must occur at least 3 times - if size < 3 { - return - } - marker = string(data[i-size : i]) - - // if this is the end marker, it must match the beginning marker - if oldmarker != "" && marker != oldmarker { - return - } - - if syntax != nil { - syn := 0 - i = skipChar(data, i, ' ') - - if i >= len(data) { - return - } - - syntaxStart := i - - if data[i] == '{' { - i++ - syntaxStart++ - - for i < len(data) && data[i] != '}' && data[i] != '\n' { - syn++ - i++ - } - - if i >= len(data) || data[i] != '}' { - return - } - - // strip all whitespace at the beginning and the end - // of the {} block - for syn > 0 && isspace(data[syntaxStart]) { - syntaxStart++ - syn-- - } - - for syn > 0 && isspace(data[syntaxStart+syn-1]) { - syn-- - } - - i++ - } else { - for i < len(data) && !isspace(data[i]) { - syn++ - i++ - } - } - - language := string(data[syntaxStart : syntaxStart+syn]) - *syntax = &language - } - - i = skipChar(data, i, ' ') - if i >= len(data) || data[i] != '\n' { - return - } - - skip = i + 1 - return -} - -func (p *parser) fencedCode(out *bytes.Buffer, data []byte, doRender bool) int { - var lang *string - beg, marker := p.isFencedCode(data, &lang, "") - if beg == 0 || beg >= len(data) { - return 0 - } - - var work bytes.Buffer - - for { - // safe to assume beg < len(data) - - // check for the end of the code block - fenceEnd, _ := p.isFencedCode(data[beg:], nil, marker) - if fenceEnd != 0 { - beg += fenceEnd - break - } - - // copy the current line - end := skipUntilChar(data, beg, '\n') + 1 - - // did we reach the end of the buffer without a closing marker? - if end >= len(data) { - return 0 - } - - // verbatim copy to the working buffer - if doRender { - work.Write(data[beg:end]) - } - beg = end - } - - syntax := "" - if lang != nil { - syntax = *lang - } - - if doRender { - p.r.BlockCode(out, work.Bytes(), syntax) - } - - return beg -} - -func (p *parser) table(out *bytes.Buffer, data []byte) int { - var header bytes.Buffer - i, columns := p.tableHeader(&header, data) - if i == 0 { - return 0 - } - - var body bytes.Buffer - - for i < len(data) { - pipes, rowStart := 0, i - for ; data[i] != '\n'; i++ { - if data[i] == '|' { - pipes++ - } - } - - if pipes == 0 { - i = rowStart - break - } - - // include the newline in data sent to tableRow - i++ - p.tableRow(&body, data[rowStart:i], columns, false) - } - - p.r.Table(out, header.Bytes(), body.Bytes(), columns) - - return i -} - -// check if the specified position is preceded by an odd number of backslashes -func isBackslashEscaped(data []byte, i int) bool { - backslashes := 0 - for i-backslashes-1 >= 0 && data[i-backslashes-1] == '\\' { - backslashes++ - } - return backslashes&1 == 1 -} - -func (p *parser) tableHeader(out *bytes.Buffer, data []byte) (size int, columns []int) { - i := 0 - colCount := 1 - for i = 0; data[i] != '\n'; i++ { - if data[i] == '|' && !isBackslashEscaped(data, i) { - colCount++ - } - } - - // doesn't look like a table header - if colCount == 1 { - return - } - - // include the newline in the data sent to tableRow - header := data[:i+1] - - // column count ignores pipes at beginning or end of line - if data[0] == '|' { - colCount-- - } - if i > 2 && data[i-1] == '|' && !isBackslashEscaped(data, i-1) { - colCount-- - } - - columns = make([]int, colCount) - - // move on to the header underline - i++ - if i >= len(data) { - return - } - - if data[i] == '|' && !isBackslashEscaped(data, i) { - i++ - } - i = skipChar(data, i, ' ') - - // each column header is of form: / *:?-+:? *|/ with # dashes + # colons >= 3 - // and trailing | optional on last column - col := 0 - for data[i] != '\n' { - dashes := 0 - - if data[i] == ':' { - i++ - columns[col] |= TABLE_ALIGNMENT_LEFT - dashes++ - } - for data[i] == '-' { - i++ - dashes++ - } - if data[i] == ':' { - i++ - columns[col] |= TABLE_ALIGNMENT_RIGHT - dashes++ - } - for data[i] == ' ' { - i++ - } - - // end of column test is messy - switch { - case dashes < 3: - // not a valid column - return - - case data[i] == '|' && !isBackslashEscaped(data, i): - // marker found, now skip past trailing whitespace - col++ - i++ - for data[i] == ' ' { - i++ - } - - // trailing junk found after last column - if col >= colCount && data[i] != '\n' { - return - } - - case (data[i] != '|' || isBackslashEscaped(data, i)) && col+1 < colCount: - // something else found where marker was required - return - - case data[i] == '\n': - // marker is optional for the last column - col++ - - default: - // trailing junk found after last column - return - } - } - if col != colCount { - return - } - - p.tableRow(out, header, columns, true) - size = i + 1 - return -} - -func (p *parser) tableRow(out *bytes.Buffer, data []byte, columns []int, header bool) { - i, col := 0, 0 - var rowWork bytes.Buffer - - if data[i] == '|' && !isBackslashEscaped(data, i) { - i++ - } - - for col = 0; col < len(columns) && i < len(data); col++ { - for data[i] == ' ' { - i++ - } - - cellStart := i - - for (data[i] != '|' || isBackslashEscaped(data, i)) && data[i] != '\n' { - i++ - } - - cellEnd := i - - // skip the end-of-cell marker, possibly taking us past end of buffer - i++ - - for cellEnd > cellStart && data[cellEnd-1] == ' ' { - cellEnd-- - } - - var cellWork bytes.Buffer - p.inline(&cellWork, data[cellStart:cellEnd]) - - if header { - p.r.TableHeaderCell(&rowWork, cellWork.Bytes(), columns[col]) - } else { - p.r.TableCell(&rowWork, cellWork.Bytes(), columns[col]) - } - } - - // pad it out with empty columns to get the right number - for ; col < len(columns); col++ { - if header { - p.r.TableHeaderCell(&rowWork, nil, columns[col]) - } else { - p.r.TableCell(&rowWork, nil, columns[col]) - } - } - - // silently ignore rows with too many cells - - p.r.TableRow(out, rowWork.Bytes()) -} - -// returns blockquote prefix length -func (p *parser) quotePrefix(data []byte) int { - i := 0 - for i < 3 && data[i] == ' ' { - i++ - } - if data[i] == '>' { - if data[i+1] == ' ' { - return i + 2 - } - return i + 1 - } - return 0 -} - -// blockquote ends with at least one blank line -// followed by something without a blockquote prefix -func (p *parser) terminateBlockquote(data []byte, beg, end int) bool { - if p.isEmpty(data[beg:]) <= 0 { - return false - } - if end >= len(data) { - return true - } - return p.quotePrefix(data[end:]) == 0 && p.isEmpty(data[end:]) == 0 -} - -// parse a blockquote fragment -func (p *parser) quote(out *bytes.Buffer, data []byte) int { - var raw bytes.Buffer - beg, end := 0, 0 - for beg < len(data) { - end = beg - // Step over whole lines, collecting them. While doing that, check for - // fenced code and if one's found, incorporate it altogether, - // irregardless of any contents inside it - for data[end] != '\n' { - if p.flags&EXTENSION_FENCED_CODE != 0 { - if i := p.fencedCode(out, data[end:], false); i > 0 { - // -1 to compensate for the extra end++ after the loop: - end += i - 1 - break - } - } - end++ - } - end++ - - if pre := p.quotePrefix(data[beg:]); pre > 0 { - // skip the prefix - beg += pre - } else if p.terminateBlockquote(data, beg, end) { - break - } - - // this line is part of the blockquote - raw.Write(data[beg:end]) - beg = end - } - - var cooked bytes.Buffer - p.block(&cooked, raw.Bytes()) - p.r.BlockQuote(out, cooked.Bytes()) - return end -} - -// returns prefix length for block code -func (p *parser) codePrefix(data []byte) int { - if data[0] == ' ' && data[1] == ' ' && data[2] == ' ' && data[3] == ' ' { - return 4 - } - return 0 -} - -func (p *parser) code(out *bytes.Buffer, data []byte) int { - var work bytes.Buffer - - i := 0 - for i < len(data) { - beg := i - for data[i] != '\n' { - i++ - } - i++ - - blankline := p.isEmpty(data[beg:i]) > 0 - if pre := p.codePrefix(data[beg:i]); pre > 0 { - beg += pre - } else if !blankline { - // non-empty, non-prefixed line breaks the pre - i = beg - break - } - - // verbatim copy to the working buffeu - if blankline { - work.WriteByte('\n') - } else { - work.Write(data[beg:i]) - } - } - - // trim all the \n off the end of work - workbytes := work.Bytes() - eol := len(workbytes) - for eol > 0 && workbytes[eol-1] == '\n' { - eol-- - } - if eol != len(workbytes) { - work.Truncate(eol) - } - - work.WriteByte('\n') - - p.r.BlockCode(out, work.Bytes(), "") - - return i -} - -// returns unordered list item prefix -func (p *parser) uliPrefix(data []byte) int { - i := 0 - - // start with up to 3 spaces - for i < 3 && data[i] == ' ' { - i++ - } - - // need a *, +, or - followed by a space - if (data[i] != '*' && data[i] != '+' && data[i] != '-') || - data[i+1] != ' ' { - return 0 - } - return i + 2 -} - -// returns ordered list item prefix -func (p *parser) oliPrefix(data []byte) int { - i := 0 - - // start with up to 3 spaces - for i < 3 && data[i] == ' ' { - i++ - } - - // count the digits - start := i - for data[i] >= '0' && data[i] <= '9' { - i++ - } - - // we need >= 1 digits followed by a dot and a space - if start == i || data[i] != '.' || data[i+1] != ' ' { - return 0 - } - return i + 2 -} - -// returns definition list item prefix -func (p *parser) dliPrefix(data []byte) int { - i := 0 - - // need a : followed by a spaces - if data[i] != ':' || data[i+1] != ' ' { - return 0 - } - for data[i] == ' ' { - i++ - } - return i + 2 -} - -// parse ordered or unordered list block -func (p *parser) list(out *bytes.Buffer, data []byte, flags int) int { - i := 0 - flags |= LIST_ITEM_BEGINNING_OF_LIST - work := func() bool { - for i < len(data) { - skip := p.listItem(out, data[i:], &flags) - i += skip - - if skip == 0 || flags&LIST_ITEM_END_OF_LIST != 0 { - break - } - flags &= ^LIST_ITEM_BEGINNING_OF_LIST - } - return true - } - - p.r.List(out, work, flags) - return i -} - -// Parse a single list item. -// Assumes initial prefix is already removed if this is a sublist. -func (p *parser) listItem(out *bytes.Buffer, data []byte, flags *int) int { - // keep track of the indentation of the first line - itemIndent := 0 - for itemIndent < 3 && data[itemIndent] == ' ' { - itemIndent++ - } - - i := p.uliPrefix(data) - if i == 0 { - i = p.oliPrefix(data) - } - if i == 0 { - i = p.dliPrefix(data) - // reset definition term flag - if i > 0 { - *flags &= ^LIST_TYPE_TERM - } - } - if i == 0 { - // if in defnition list, set term flag and continue - if *flags&LIST_TYPE_DEFINITION != 0 { - *flags |= LIST_TYPE_TERM - } else { - return 0 - } - } - - // skip leading whitespace on first line - for data[i] == ' ' { - i++ - } - - // find the end of the line - line := i - for i > 0 && data[i-1] != '\n' { - i++ - } - - // get working buffer - var raw bytes.Buffer - - // put the first line into the working buffer - raw.Write(data[line:i]) - line = i - - // process the following lines - containsBlankLine := false - sublist := 0 - -gatherlines: - for line < len(data) { - i++ - - // find the end of this line - for data[i-1] != '\n' { - i++ - } - - // if it is an empty line, guess that it is part of this item - // and move on to the next line - if p.isEmpty(data[line:i]) > 0 { - containsBlankLine = true - raw.Write(data[line:i]) - line = i - continue - } - - // calculate the indentation - indent := 0 - for indent < 4 && line+indent < i && data[line+indent] == ' ' { - indent++ - } - - chunk := data[line+indent : i] - - // evaluate how this line fits in - switch { - // is this a nested list item? - case (p.uliPrefix(chunk) > 0 && !p.isHRule(chunk)) || - p.oliPrefix(chunk) > 0 || - p.dliPrefix(chunk) > 0: - - if containsBlankLine { - *flags |= LIST_ITEM_CONTAINS_BLOCK - } - - // to be a nested list, it must be indented more - // if not, it is the next item in the same list - if indent <= itemIndent { - break gatherlines - } - - // is this the first item in the nested list? - if sublist == 0 { - sublist = raw.Len() - } - - // is this a nested prefix header? - case p.isPrefixHeader(chunk): - // if the header is not indented, it is not nested in the list - // and thus ends the list - if containsBlankLine && indent < 4 { - *flags |= LIST_ITEM_END_OF_LIST - break gatherlines - } - *flags |= LIST_ITEM_CONTAINS_BLOCK - - // anything following an empty line is only part - // of this item if it is indented 4 spaces - // (regardless of the indentation of the beginning of the item) - case containsBlankLine && indent < 4: - if *flags&LIST_TYPE_DEFINITION != 0 && i < len(data)-1 { - // is the next item still a part of this list? - next := i - for data[next] != '\n' { - next++ - } - for next < len(data)-1 && data[next] == '\n' { - next++ - } - if i < len(data)-1 && data[i] != ':' && data[next] != ':' { - *flags |= LIST_ITEM_END_OF_LIST - } - } else { - *flags |= LIST_ITEM_END_OF_LIST - } - break gatherlines - - // a blank line means this should be parsed as a block - case containsBlankLine: - *flags |= LIST_ITEM_CONTAINS_BLOCK - } - - containsBlankLine = false - - // add the line into the working buffer without prefix - raw.Write(data[line+indent : i]) - - line = i - } - - rawBytes := raw.Bytes() - - // render the contents of the list item - var cooked bytes.Buffer - if *flags&LIST_ITEM_CONTAINS_BLOCK != 0 && *flags&LIST_TYPE_TERM == 0 { - // intermediate render of block item, except for definition term - if sublist > 0 { - p.block(&cooked, rawBytes[:sublist]) - p.block(&cooked, rawBytes[sublist:]) - } else { - p.block(&cooked, rawBytes) - } - } else { - // intermediate render of inline item - if sublist > 0 { - p.inline(&cooked, rawBytes[:sublist]) - p.block(&cooked, rawBytes[sublist:]) - } else { - p.inline(&cooked, rawBytes) - } - } - - // render the actual list item - cookedBytes := cooked.Bytes() - parsedEnd := len(cookedBytes) - - // strip trailing newlines - for parsedEnd > 0 && cookedBytes[parsedEnd-1] == '\n' { - parsedEnd-- - } - p.r.ListItem(out, cookedBytes[:parsedEnd], *flags) - - return line -} - -// render a single paragraph that has already been parsed out -func (p *parser) renderParagraph(out *bytes.Buffer, data []byte) { - if len(data) == 0 { - return - } - - // trim leading spaces - beg := 0 - for data[beg] == ' ' { - beg++ - } - - // trim trailing newline - end := len(data) - 1 - - // trim trailing spaces - for end > beg && data[end-1] == ' ' { - end-- - } - - work := func() bool { - p.inline(out, data[beg:end]) - return true - } - p.r.Paragraph(out, work) -} - -func (p *parser) paragraph(out *bytes.Buffer, data []byte) int { - // prev: index of 1st char of previous line - // line: index of 1st char of current line - // i: index of cursor/end of current line - var prev, line, i int - - // keep going until we find something to mark the end of the paragraph - for i < len(data) { - // mark the beginning of the current line - prev = line - current := data[i:] - line = i - - // did we find a blank line marking the end of the paragraph? - if n := p.isEmpty(current); n > 0 { - // did this blank line followed by a definition list item? - if p.flags&EXTENSION_DEFINITION_LISTS != 0 { - if i < len(data)-1 && data[i+1] == ':' { - return p.list(out, data[prev:], LIST_TYPE_DEFINITION) - } - } - - p.renderParagraph(out, data[:i]) - return i + n - } - - // an underline under some text marks a header, so our paragraph ended on prev line - if i > 0 { - if level := p.isUnderlinedHeader(current); level > 0 { - // render the paragraph - p.renderParagraph(out, data[:prev]) - - // ignore leading and trailing whitespace - eol := i - 1 - for prev < eol && data[prev] == ' ' { - prev++ - } - for eol > prev && data[eol-1] == ' ' { - eol-- - } - - // render the header - // this ugly double closure avoids forcing variables onto the heap - work := func(o *bytes.Buffer, pp *parser, d []byte) func() bool { - return func() bool { - pp.inline(o, d) - return true - } - }(out, p, data[prev:eol]) - - id := "" - if p.flags&EXTENSION_AUTO_HEADER_IDS != 0 { - id = sanitized_anchor_name.Create(string(data[prev:eol])) - } - - p.r.Header(out, work, level, id) - - // find the end of the underline - for data[i] != '\n' { - i++ - } - return i - } - } - - // if the next line starts a block of HTML, then the paragraph ends here - if p.flags&EXTENSION_LAX_HTML_BLOCKS != 0 { - if data[i] == '<' && p.html(out, current, false) > 0 { - // rewind to before the HTML block - p.renderParagraph(out, data[:i]) - return i - } - } - - // if there's a prefixed header or a horizontal rule after this, paragraph is over - if p.isPrefixHeader(current) || p.isHRule(current) { - p.renderParagraph(out, data[:i]) - return i - } - - // if there's a fenced code block, paragraph is over - if p.flags&EXTENSION_FENCED_CODE != 0 { - if p.fencedCode(out, current, false) > 0 { - p.renderParagraph(out, data[:i]) - return i - } - } - - // if there's a definition list item, prev line is a definition term - if p.flags&EXTENSION_DEFINITION_LISTS != 0 { - if p.dliPrefix(current) != 0 { - return p.list(out, data[prev:], LIST_TYPE_DEFINITION) - } - } - - // if there's a list after this, paragraph is over - if p.flags&EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK != 0 { - if p.uliPrefix(current) != 0 || - p.oliPrefix(current) != 0 || - p.quotePrefix(current) != 0 || - p.codePrefix(current) != 0 { - p.renderParagraph(out, data[:i]) - return i - } - } - - // otherwise, scan to the beginning of the next line - for data[i] != '\n' { - i++ - } - i++ - } - - p.renderParagraph(out, data[:i]) - return i -} diff --git a/vendor/github.com/russross/blackfriday/block_test.go b/vendor/github.com/russross/blackfriday/block_test.go deleted file mode 100644 index f59268ee..00000000 --- a/vendor/github.com/russross/blackfriday/block_test.go +++ /dev/null @@ -1,1621 +0,0 @@ -// -// Blackfriday Markdown Processor -// Available at http://github.com/russross/blackfriday -// -// Copyright © 2011 Russ Ross . -// Distributed under the Simplified BSD License. -// See README.md for details. -// - -// -// Unit tests for block parsing -// - -package blackfriday - -import ( - "strings" - "testing" -) - -func runMarkdownBlockWithRenderer(input string, extensions int, renderer Renderer) string { - return string(Markdown([]byte(input), renderer, extensions)) -} - -func runMarkdownBlock(input string, extensions int) string { - htmlFlags := 0 - htmlFlags |= HTML_USE_XHTML - - renderer := HtmlRenderer(htmlFlags, "", "") - - return runMarkdownBlockWithRenderer(input, extensions, renderer) -} - -func runnerWithRendererParameters(parameters HtmlRendererParameters) func(string, int) string { - return func(input string, extensions int) string { - htmlFlags := 0 - htmlFlags |= HTML_USE_XHTML - - renderer := HtmlRendererWithParameters(htmlFlags, "", "", parameters) - - return runMarkdownBlockWithRenderer(input, extensions, renderer) - } -} - -func doTestsBlock(t *testing.T, tests []string, extensions int) { - doTestsBlockWithRunner(t, tests, extensions, runMarkdownBlock) -} - -func doTestsBlockWithRunner(t *testing.T, tests []string, extensions int, runner func(string, int) string) { - // catch and report panics - var candidate string - defer func() { - if err := recover(); err != nil { - t.Errorf("\npanic while processing [%#v]: %s\n", candidate, err) - } - }() - - for i := 0; i+1 < len(tests); i += 2 { - input := tests[i] - candidate = input - expected := tests[i+1] - actual := runner(candidate, extensions) - if actual != expected { - t.Errorf("\nInput [%#v]\nExpected[%#v]\nActual [%#v]", - candidate, expected, actual) - } - - // now test every substring to stress test bounds checking - if !testing.Short() { - for start := 0; start < len(input); start++ { - for end := start + 1; end <= len(input); end++ { - candidate = input[start:end] - _ = runMarkdownBlock(candidate, extensions) - } - } - } - } -} - -func TestPrefixHeaderNoExtensions(t *testing.T) { - var tests = []string{ - "# Header 1\n", - "

Header 1

\n", - - "## Header 2\n", - "

Header 2

\n", - - "### Header 3\n", - "

Header 3

\n", - - "#### Header 4\n", - "

Header 4

\n", - - "##### Header 5\n", - "
Header 5
\n", - - "###### Header 6\n", - "
Header 6
\n", - - "####### Header 7\n", - "
# Header 7
\n", - - "#Header 1\n", - "

Header 1

\n", - - "##Header 2\n", - "

Header 2

\n", - - "###Header 3\n", - "

Header 3

\n", - - "####Header 4\n", - "

Header 4

\n", - - "#####Header 5\n", - "
Header 5
\n", - - "######Header 6\n", - "
Header 6
\n", - - "#######Header 7\n", - "
#Header 7
\n", - - "Hello\n# Header 1\nGoodbye\n", - "

Hello

\n\n

Header 1

\n\n

Goodbye

\n", - - "* List\n# Header\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n#Header\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n * Nested list\n # Nested header\n", - "
    \n

  • List

    \n\n
      \n

    • Nested list

      \n\n" + - "

      Nested header

      • \n
    • \n
\n", - - "#Header 1 \\#\n", - "

Header 1 #

\n", - - "#Header 1 \\# foo\n", - "

Header 1 # foo

\n", - - "#Header 1 #\\##\n", - "

Header 1 ##

\n", - } - doTestsBlock(t, tests, 0) -} - -func TestPrefixHeaderSpaceExtension(t *testing.T) { - var tests = []string{ - "# Header 1\n", - "

Header 1

\n", - - "## Header 2\n", - "

Header 2

\n", - - "### Header 3\n", - "

Header 3

\n", - - "#### Header 4\n", - "

Header 4

\n", - - "##### Header 5\n", - "
Header 5
\n", - - "###### Header 6\n", - "
Header 6
\n", - - "####### Header 7\n", - "

####### Header 7

\n", - - "#Header 1\n", - "

#Header 1

\n", - - "##Header 2\n", - "

##Header 2

\n", - - "###Header 3\n", - "

###Header 3

\n", - - "####Header 4\n", - "

####Header 4

\n", - - "#####Header 5\n", - "

#####Header 5

\n", - - "######Header 6\n", - "

######Header 6

\n", - - "#######Header 7\n", - "

#######Header 7

\n", - - "Hello\n# Header 1\nGoodbye\n", - "

Hello

\n\n

Header 1

\n\n

Goodbye

\n", - - "* List\n# Header\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n#Header\n* List\n", - "
    \n
  • List\n#Header
    • \n
  • List
    • \n
\n", - - "* List\n * Nested list\n # Nested header\n", - "
    \n

  • List

    \n\n
      \n

    • Nested list

      \n\n" + - "

      Nested header

      • \n
    • \n
\n", - } - doTestsBlock(t, tests, EXTENSION_SPACE_HEADERS) -} - -func TestPrefixHeaderIdExtension(t *testing.T) { - var tests = []string{ - "# Header 1 {#someid}\n", - "

Header 1

\n", - - "# Header 1 {#someid} \n", - "

Header 1

\n", - - "# Header 1 {#someid}\n", - "

Header 1

\n", - - "# Header 1 {#someid\n", - "

Header 1 {#someid

\n", - - "# Header 1 {#someid\n", - "

Header 1 {#someid

\n", - - "# Header 1 {#someid}}\n", - "

Header 1

\n\n

}

\n", - - "## Header 2 {#someid}\n", - "

Header 2

\n", - - "### Header 3 {#someid}\n", - "

Header 3

\n", - - "#### Header 4 {#someid}\n", - "

Header 4

\n", - - "##### Header 5 {#someid}\n", - "
Header 5
\n", - - "###### Header 6 {#someid}\n", - "
Header 6
\n", - - "####### Header 7 {#someid}\n", - "
# Header 7
\n", - - "# Header 1 # {#someid}\n", - "

Header 1

\n", - - "## Header 2 ## {#someid}\n", - "

Header 2

\n", - - "Hello\n# Header 1\nGoodbye\n", - "

Hello

\n\n

Header 1

\n\n

Goodbye

\n", - - "* List\n# Header {#someid}\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n#Header {#someid}\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n * Nested list\n # Nested header {#someid}\n", - "
    \n

  • List

    \n\n
      \n

    • Nested list

      \n\n" + - "

      Nested header

      • \n
    • \n
\n", - } - doTestsBlock(t, tests, EXTENSION_HEADER_IDS) -} - -func TestPrefixHeaderIdExtensionWithPrefixAndSuffix(t *testing.T) { - var tests = []string{ - "# header 1 {#someid}\n", - "

header 1

\n", - - "## header 2 {#someid}\n", - "

header 2

\n", - - "### header 3 {#someid}\n", - "

header 3

\n", - - "#### header 4 {#someid}\n", - "

header 4

\n", - - "##### header 5 {#someid}\n", - "
header 5
\n", - - "###### header 6 {#someid}\n", - "
header 6
\n", - - "####### header 7 {#someid}\n", - "
# header 7
\n", - - "# header 1 # {#someid}\n", - "

header 1

\n", - - "## header 2 ## {#someid}\n", - "

header 2

\n", - - "* List\n# Header {#someid}\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n#Header {#someid}\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n * Nested list\n # Nested header {#someid}\n", - "
    \n

  • List

    \n\n
      \n

    • Nested list

      \n\n" + - "

      Nested header

      • \n
    • \n
\n", - } - - parameters := HtmlRendererParameters{ - HeaderIDPrefix: "PRE:", - HeaderIDSuffix: ":POST", - } - - doTestsBlockWithRunner(t, tests, EXTENSION_HEADER_IDS, runnerWithRendererParameters(parameters)) -} - -func TestPrefixAutoHeaderIdExtension(t *testing.T) { - var tests = []string{ - "# Header 1\n", - "

Header 1

\n", - - "# Header 1 \n", - "

Header 1

\n", - - "## Header 2\n", - "

Header 2

\n", - - "### Header 3\n", - "

Header 3

\n", - - "#### Header 4\n", - "

Header 4

\n", - - "##### Header 5\n", - "
Header 5
\n", - - "###### Header 6\n", - "
Header 6
\n", - - "####### Header 7\n", - "
# Header 7
\n", - - "Hello\n# Header 1\nGoodbye\n", - "

Hello

\n\n

Header 1

\n\n

Goodbye

\n", - - "* List\n# Header\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n#Header\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n * Nested list\n # Nested header\n", - "
    \n

  • List

    \n\n
      \n

    • Nested list

      \n\n" + - "

      Nested header

      • \n
    • \n
\n", - - "# Header\n\n# Header\n", - "

Header

\n\n

Header

\n", - - "# Header 1\n\n# Header 1", - "

Header 1

\n\n

Header 1

\n", - - "# Header\n\n# Header 1\n\n# Header\n\n# Header", - "

Header

\n\n

Header 1

\n\n

Header

\n\n

Header

\n", - } - doTestsBlock(t, tests, EXTENSION_AUTO_HEADER_IDS) -} - -func TestPrefixAutoHeaderIdExtensionWithPrefixAndSuffix(t *testing.T) { - var tests = []string{ - "# Header 1\n", - "

Header 1

\n", - - "# Header 1 \n", - "

Header 1

\n", - - "## Header 2\n", - "

Header 2

\n", - - "### Header 3\n", - "

Header 3

\n", - - "#### Header 4\n", - "

Header 4

\n", - - "##### Header 5\n", - "
Header 5
\n", - - "###### Header 6\n", - "
Header 6
\n", - - "####### Header 7\n", - "
# Header 7
\n", - - "Hello\n# Header 1\nGoodbye\n", - "

Hello

\n\n

Header 1

\n\n

Goodbye

\n", - - "* List\n# Header\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n#Header\n* List\n", - "
    \n

  • List

    \n\n

    Header

    • \n\n

  • List

    • \n
\n", - - "* List\n * Nested list\n # Nested header\n", - "
    \n

  • List

    \n\n
      \n

    • Nested list

      \n\n" + - "

      Nested header

      • \n
    • \n
\n", - - "# Header\n\n# Header\n", - "

Header

\n\n

Header

\n", - - "# Header 1\n\n# Header 1", - "

Header 1

\n\n

Header 1

\n", - - "# Header\n\n# Header 1\n\n# Header\n\n# Header", - "

Header

\n\n

Header 1

\n\n

Header

\n\n

Header

\n", - } - - parameters := HtmlRendererParameters{ - HeaderIDPrefix: "PRE:", - HeaderIDSuffix: ":POST", - } - - doTestsBlockWithRunner(t, tests, EXTENSION_AUTO_HEADER_IDS, runnerWithRendererParameters(parameters)) -} - -func TestPrefixMultipleHeaderExtensions(t *testing.T) { - var tests = []string{ - "# Header\n\n# Header {#header}\n\n# Header 1", - "

Header

\n\n

Header

\n\n

Header 1

\n", - } - doTestsBlock(t, tests, EXTENSION_AUTO_HEADER_IDS|EXTENSION_HEADER_IDS) -} - -func TestUnderlineHeaders(t *testing.T) { - var tests = []string{ - "Header 1\n========\n", - "

Header 1

\n", - - "Header 2\n--------\n", - "

Header 2

\n", - - "A\n=\n", - "

A

\n", - - "B\n-\n", - "

B

\n", - - "Paragraph\nHeader\n=\n", - "

Paragraph

\n\n

Header

\n", - - "Header\n===\nParagraph\n", - "

Header

\n\n

Paragraph

\n", - - "Header\n===\nAnother header\n---\n", - "

Header

\n\n

Another header

\n", - - " Header\n======\n", - "

Header

\n", - - " Code\n========\n", - "
Code\n
\n\n

========

\n", - - "Header with *inline*\n=====\n", - "

Header with inline

\n", - - "* List\n * Sublist\n Not a header\n ------\n", - "
    \n
  • List\n\n
      \n
    • Sublist\nNot a header\n------
      • \n
    • \n
\n", - - "Paragraph\n\n\n\n\nHeader\n===\n", - "

Paragraph

\n\n

Header

\n", - - "Trailing space \n==== \n\n", - "

Trailing space

\n", - - "Trailing spaces\n==== \n\n", - "

Trailing spaces

\n", - - "Double underline\n=====\n=====\n", - "

Double underline

\n\n

=====

\n", - } - doTestsBlock(t, tests, 0) -} - -func TestUnderlineHeadersAutoIDs(t *testing.T) { - var tests = []string{ - "Header 1\n========\n", - "

Header 1

\n", - - "Header 2\n--------\n", - "

Header 2

\n", - - "A\n=\n", - "

A

\n", - - "B\n-\n", - "

B

\n", - - "Paragraph\nHeader\n=\n", - "

Paragraph

\n\n

Header

\n", - - "Header\n===\nParagraph\n", - "

Header

\n\n

Paragraph

\n", - - "Header\n===\nAnother header\n---\n", - "

Header

\n\n

Another header

\n", - - " Header\n======\n", - "

Header

\n", - - "Header with *inline*\n=====\n", - "

Header with inline

\n", - - "Paragraph\n\n\n\n\nHeader\n===\n", - "

Paragraph

\n\n

Header

\n", - - "Trailing space \n==== \n\n", - "

Trailing space

\n", - - "Trailing spaces\n==== \n\n", - "

Trailing spaces

\n", - - "Double underline\n=====\n=====\n", - "

Double underline

\n\n

=====

\n", - - "Header\n======\n\nHeader\n======\n", - "

Header

\n\n

Header

\n", - - "Header 1\n========\n\nHeader 1\n========\n", - "

Header 1

\n\n

Header 1

\n", - } - doTestsBlock(t, tests, EXTENSION_AUTO_HEADER_IDS) -} - -func TestHorizontalRule(t *testing.T) { - var tests = []string{ - "-\n", - "

-

\n", - - "--\n", - "

--

\n", - - "---\n", - "
\n", - - "----\n", - "
\n", - - "*\n", - "

*

\n", - - "**\n", - "

**

\n", - - "***\n", - "
\n", - - "****\n", - "
\n", - - "_\n", - "

_

\n", - - "__\n", - "

__

\n", - - "___\n", - "
\n", - - "____\n", - "
\n", - - "-*-\n", - "

-*-

\n", - - "- - -\n", - "
\n", - - "* * *\n", - "
\n", - - "_ _ _\n", - "
\n", - - "-----*\n", - "

-----*

\n", - - " ------ \n", - "
\n", - - "Hello\n***\n", - "

Hello

\n\n
\n", - - "---\n***\n___\n", - "
\n\n
\n\n
\n", - } - doTestsBlock(t, tests, 0) -} - -func TestUnorderedList(t *testing.T) { - var tests = []string{ - "* Hello\n", - "
    \n
  • Hello
    • \n
\n", - - "* Yin\n* Yang\n", - "
    \n
  • Yin
    • \n
  • Yang
    • \n
\n", - - "* Ting\n* Bong\n* Goo\n", - "
    \n
  • Ting
    • \n
  • Bong
    • \n
  • Goo
    • \n
\n", - - "* Yin\n\n* Yang\n", - "
    \n

  • Yin

    • \n\n

  • Yang

    • \n
\n", - - "* Ting\n\n* Bong\n* Goo\n", - "
    \n

  • Ting

    • \n\n

  • Bong

    • \n\n

  • Goo

    • \n
\n", - - "+ Hello\n", - "
    \n
  • Hello
    • \n
\n", - - "+ Yin\n+ Yang\n", - "
    \n
  • Yin
    • \n
  • Yang
    • \n
\n", - - "+ Ting\n+ Bong\n+ Goo\n", - "
    \n
  • Ting
    • \n
  • Bong
    • \n
  • Goo
    • \n
\n", - - "+ Yin\n\n+ Yang\n", - "
    \n

  • Yin

    • \n\n

  • Yang

    • \n
\n", - - "+ Ting\n\n+ Bong\n+ Goo\n", - "
    \n

  • Ting

    • \n\n

  • Bong

    • \n\n

  • Goo

    • \n
\n", - - "- Hello\n", - "
    \n
  • Hello
    • \n
\n", - - "- Yin\n- Yang\n", - "
    \n
  • Yin
    • \n
  • Yang
    • \n
\n", - - "- Ting\n- Bong\n- Goo\n", - "
    \n
  • Ting
    • \n
  • Bong
    • \n
  • Goo
    • \n
\n", - - "- Yin\n\n- Yang\n", - "
    \n

  • Yin

    • \n\n

  • Yang

    • \n
\n", - - "- Ting\n\n- Bong\n- Goo\n", - "
    \n

  • Ting

    • \n\n

  • Bong

    • \n\n

  • Goo

    • \n
\n", - - "*Hello\n", - "

*Hello

\n", - - "* Hello \n", - "
    \n
  • Hello
    • \n
\n", - - "* Hello \n Next line \n", - "
    \n
  • Hello\nNext line
    • \n
\n", - - "Paragraph\n* No linebreak\n", - "

Paragraph\n* No linebreak

\n", - - "Paragraph\n\n* Linebreak\n", - "

Paragraph

\n\n
    \n
  • Linebreak
    • \n
\n", - - "* List\n * Nested list\n", - "
    \n
  • List\n\n
      \n
    • Nested list
      • \n
    • \n
\n", - - "* List\n\n * Nested list\n", - "
    \n

  • List

    \n\n
      \n
    • Nested list
      • \n
    • \n
\n", - - "* List\n Second line\n\n + Nested\n", - "
    \n

  • List\nSecond line

    \n\n
      \n
    • Nested
      • \n
    • \n
\n", - - "* List\n + Nested\n\n Continued\n", - "
    \n

  • List

    \n\n
      \n
    • Nested
      • \n
    \n\n

    Continued

    • \n
\n", - - "* List\n * shallow indent\n", - "
    \n
  • List\n\n
      \n
    • shallow indent
      • \n
    • \n
\n", - - "* List\n" + - " * shallow indent\n" + - " * part of second list\n" + - " * still second\n" + - " * almost there\n" + - " * third level\n", - "
    \n" + - "
  • List\n\n" + - "
      \n" + - "
    • shallow indent
      • \n" + - "
    • part of second list
      • \n" + - "
    • still second
      • \n" + - "
    • almost there\n\n" + - "
        \n" + - "
      • third level
        • \n" + - "
      • \n" + - "
    • \n" + - "
\n", - - "* List\n extra indent, same paragraph\n", - "
    \n
  • List\n extra indent, same paragraph
    • \n
\n", - - "* List\n\n code block\n", - "
    \n

  • List

    \n\n
    code block\n
    • \n
\n", - - "* List\n\n code block with spaces\n", - "
    \n

  • List

    \n\n
      code block with spaces\n
    • \n
\n", - - "* List\n\n * sublist\n\n normal text\n\n * another sublist\n", - "
    \n

  • List

    \n\n
      \n
    • sublist
      • \n
    \n\n

    normal text

    \n\n
      \n
    • another sublist
      • \n
    • \n
\n", - - `* Foo - - bar - - qux -`, - `
    -

  • Foo

    - -
    bar
-
-qux
-
    • -
-`, - } - doTestsBlock(t, tests, 0) -} - -func TestFencedCodeBlockWithinList(t *testing.T) { - doTestsBlock(t, []string{ - "* Foo\n\n ```\n bar\n\n qux\n ```\n", - `
    -

  • Foo

    - -
    bar
-
-qux
-
    • -
-`, - }, EXTENSION_FENCED_CODE) -} - -func TestOrderedList(t *testing.T) { - var tests = []string{ - "1. Hello\n", - "
    \n
  1. Hello
    2. \n
\n", - - "1. Yin\n2. Yang\n", - "
    \n
  1. Yin
    2. \n
  2. Yang
    3. \n
\n", - - "1. Ting\n2. Bong\n3. Goo\n", - "
    \n
  1. Ting
    2. \n
  2. Bong
    3. \n
  3. Goo
    4. \n
\n", - - "1. Yin\n\n2. Yang\n", - "
    \n

  1. Yin

    2. \n\n

  2. Yang

    3. \n
\n", - - "1. Ting\n\n2. Bong\n3. Goo\n", - "
    \n

  1. Ting

    2. \n\n

  2. Bong

    3. \n\n

  3. Goo

    4. \n
\n", - - "1 Hello\n", - "

1 Hello

\n", - - "1.Hello\n", - "

1.Hello

\n", - - "1. Hello \n", - "
    \n
  1. Hello
    2. \n
\n", - - "1. Hello \n Next line \n", - "
    \n
  1. Hello\nNext line
    2. \n
\n", - - "Paragraph\n1. No linebreak\n", - "

Paragraph\n1. No linebreak

\n", - - "Paragraph\n\n1. Linebreak\n", - "

Paragraph

\n\n
    \n
  1. Linebreak
    2. \n
\n", - - "1. List\n 1. Nested list\n", - "
    \n
  1. List\n\n
      \n
    1. Nested list
      2. \n
    2. \n
\n", - - "1. List\n\n 1. Nested list\n", - "
    \n

  1. List

    \n\n
      \n
    1. Nested list
      2. \n
    2. \n
\n", - - "1. List\n Second line\n\n 1. Nested\n", - "
    \n

  1. List\nSecond line

    \n\n
      \n
    1. Nested
      2. \n
    2. \n
\n", - - "1. List\n 1. Nested\n\n Continued\n", - "
    \n

  1. List

    \n\n
      \n
    1. Nested
      2. \n
    \n\n

    Continued

    2. \n
\n", - - "1. List\n 1. shallow indent\n", - "
    \n
  1. List\n\n
      \n
    1. shallow indent
      2. \n
    2. \n
\n", - - "1. List\n" + - " 1. shallow indent\n" + - " 2. part of second list\n" + - " 3. still second\n" + - " 4. almost there\n" + - " 1. third level\n", - "
    \n" + - "
  1. List\n\n" + - "
      \n" + - "
    1. shallow indent
      2. \n" + - "
    2. part of second list
      3. \n" + - "
    3. still second
      4. \n" + - "
    4. almost there\n\n" + - "
        \n" + - "
      1. third level
        2. \n" + - "
      5. \n" + - "
    2. \n" + - "
\n", - - "1. List\n extra indent, same paragraph\n", - "
    \n
  1. List\n extra indent, same paragraph
    2. \n
\n", - - "1. List\n\n code block\n", - "
    \n

  1. List

    \n\n
    code block\n
    2. \n
\n", - - "1. List\n\n code block with spaces\n", - "
    \n

  1. List

    \n\n
      code block with spaces\n
    2. \n
\n", - - "1. List\n * Mixted list\n", - "
    \n
  1. List\n\n
      \n
    • Mixted list
      • \n
    2. \n
\n", - - "1. List\n * Mixed list\n", - "
    \n
  1. List\n\n
      \n
    • Mixed list
      • \n
    2. \n
\n", - - "* Start with unordered\n 1. Ordered\n", - "
    \n
  • Start with unordered\n\n
      \n
    1. Ordered
      2. \n
    • \n
\n", - - "* Start with unordered\n 1. Ordered\n", - "
    \n
  • Start with unordered\n\n
      \n
    1. Ordered
      2. \n
    • \n
\n", - - "1. numbers\n1. are ignored\n", - "
    \n
  1. numbers
    2. \n
  2. are ignored
    3. \n
\n", - - `1. Foo - - bar - - - - qux -`, - `
    -

  1. Foo

    - -
    bar
-
-
-
-qux
-
    2. -
-`, - } - doTestsBlock(t, tests, 0) -} - -func TestDefinitionList(t *testing.T) { - var tests = []string{ - "Term 1\n: Definition a\n", - "
\n
Term 1
\n
Definition a
\n
\n", - - "Term 1\n: Definition a \n", - "
\n
Term 1
\n
Definition a
\n
\n", - - "Term 1\n: Definition a\n: Definition b\n", - "
\n
Term 1
\n
Definition a
\n
Definition b
\n
\n", - - "Term 1\n: Definition a\n\nTerm 2\n: Definition b\n", - "
\n" + - "
Term 1
\n" + - "
Definition a
\n" + - "
Term 2
\n" + - "
Definition b
\n" + - "
\n", - - "Term 1\n: Definition a\n\nTerm 2\n: Definition b\n\nTerm 3\n: Definition c\n", - "
\n" + - "
Term 1
\n" + - "
Definition a
\n" + - "
Term 2
\n" + - "
Definition b
\n" + - "
Term 3
\n" + - "
Definition c
\n" + - "
\n", - - "Term 1\n: Definition a\n: Definition b\n\nTerm 2\n: Definition c\n", - "
\n" + - "
Term 1
\n" + - "
Definition a
\n" + - "
Definition b
\n" + - "
Term 2
\n" + - "
Definition c
\n" + - "
\n", - - "Term 1\n\n: Definition a\n\nTerm 2\n\n: Definition b\n", - "
\n" + - "
Term 1
\n" + - "

Definition a

\n" + - "
Term 2
\n" + - "

Definition b

\n" + - "
\n", - - "Term 1\n\n: Definition a\n\n: Definition b\n\nTerm 2\n\n: Definition c\n", - "
\n" + - "
Term 1
\n" + - "

Definition a

\n" + - "

Definition b

\n" + - "
Term 2
\n" + - "

Definition c

\n" + - "
\n", - - "Term 1\n: Definition a\nNext line\n", - "
\n
Term 1
\n
Definition a\nNext line
\n
\n", - - "Term 1\n: Definition a\n Next line\n", - "
\n
Term 1
\n
Definition a\nNext line
\n
\n", - - "Term 1\n: Definition a \n Next line \n", - "
\n
Term 1
\n
Definition a\nNext line
\n
\n", - - "Term 1\n: Definition a\nNext line\n\nTerm 2\n: Definition b", - "
\n" + - "
Term 1
\n" + - "
Definition a\nNext line
\n" + - "
Term 2
\n" + - "
Definition b
\n" + - "
\n", - - "Term 1\n: Definition a\n", - "
\n
Term 1
\n
Definition a
\n
\n", - - "Term 1\n:Definition a\n", - "

Term 1\n:Definition a

\n", - - "Term 1\n\n: Definition a\n\nTerm 2\n\n: Definition b\n\nText 1", - "
\n" + - "
Term 1
\n" + - "

Definition a

\n" + - "
Term 2
\n" + - "

Definition b

\n" + - "
\n" + - "\n

Text 1

\n", - - "Term 1\n\n: Definition a\n\nText 1\n\nTerm 2\n\n: Definition b\n\nText 2", - "
\n" + - "
Term 1
\n" + - "

Definition a

\n" + - "
\n" + - "\n

Text 1

\n" + - "\n
\n" + - "
Term 2
\n" + - "

Definition b

\n" + - "
\n" + - "\n

Text 2

\n", - } - doTestsBlock(t, tests, EXTENSION_DEFINITION_LISTS) -} - -func TestPreformattedHtml(t *testing.T) { - var tests = []string{ - "
\n", - "
\n", - - "
\n
\n", - "
\n
\n", - - "
\n
\nParagraph\n", - "

\n
\nParagraph

\n", - - "
\n
\n", - "
\n
\n", - - "
\nAnything here\n
\n", - "
\nAnything here\n
\n", - - "
\n Anything here\n
\n", - "
\n Anything here\n
\n", - - "
\nAnything here\n
\n", - "
\nAnything here\n
\n", - - "
\nThis is *not* &proceessed\n
\n", - "
\nThis is *not* &proceessed\n
\n", - - "\n Something\n\n", - "

\n Something\n

\n", - - "
\n Something here\n\n", - "

\n Something here\n

\n", - - "Paragraph\n
\nHere? >&<\n
\n", - "

Paragraph\n

\nHere? >&<\n

\n", - - "Paragraph\n\n
\nHow about here? >&<\n
\n", - "

Paragraph

\n\n
\nHow about here? >&<\n
\n", - - "Paragraph\n
\nHere? >&<\n
\nAnd here?\n", - "

Paragraph\n

\nHere? >&<\n
\nAnd here?

\n", - - "Paragraph\n\n
\nHow about here? >&<\n
\nAnd here?\n", - "

Paragraph

\n\n

\nHow about here? >&<\n
\nAnd here?

\n", - - "Paragraph\n
\nHere? >&<\n
\n\nAnd here?\n", - "

Paragraph\n

\nHere? >&<\n

\n\n

And here?

\n", - - "Paragraph\n\n
\nHow about here? >&<\n
\n\nAnd here?\n", - "

Paragraph

\n\n
\nHow about here? >&<\n
\n\n

And here?

\n", - } - doTestsBlock(t, tests, 0) -} - -func TestPreformattedHtmlLax(t *testing.T) { - var tests = []string{ - "Paragraph\n
\nHere? >&<\n
\n", - "

Paragraph

\n\n
\nHere? >&<\n
\n", - - "Paragraph\n\n
\nHow about here? >&<\n
\n", - "

Paragraph

\n\n
\nHow about here? >&<\n
\n", - - "Paragraph\n
\nHere? >&<\n
\nAnd here?\n", - "

Paragraph

\n\n
\nHere? >&<\n
\n\n

And here?

\n", - - "Paragraph\n\n
\nHow about here? >&<\n
\nAnd here?\n", - "

Paragraph

\n\n
\nHow about here? >&<\n
\n\n

And here?

\n", - - "Paragraph\n
\nHere? >&<\n
\n\nAnd here?\n", - "

Paragraph

\n\n
\nHere? >&<\n
\n\n

And here?

\n", - - "Paragraph\n\n
\nHow about here? >&<\n
\n\nAnd here?\n", - "

Paragraph

\n\n
\nHow about here? >&<\n
\n\n

And here?

\n", - } - doTestsBlock(t, tests, EXTENSION_LAX_HTML_BLOCKS) -} - -func TestFencedCodeBlock(t *testing.T) { - var tests = []string{ - "``` go\nfunc foo() bool {\n\treturn true;\n}\n```\n", - "
func foo() bool {\n\treturn true;\n}\n
\n", - - "``` c\n/* special & char < > \" escaping */\n```\n", - "
/* special & char < > " escaping */\n
\n", - - "``` c\nno *inline* processing ~~of text~~\n```\n", - "
no *inline* processing ~~of text~~\n
\n", - - "```\nNo language\n```\n", - "
No language\n
\n", - - "``` {ocaml}\nlanguage in braces\n```\n", - "
language in braces\n
\n", - - "``` {ocaml} \nwith extra whitespace\n```\n", - "
with extra whitespace\n
\n", - - "```{ ocaml }\nwith extra whitespace\n```\n", - "
with extra whitespace\n
\n", - - "~ ~~ java\nWith whitespace\n~~~\n", - "

~ ~~ java\nWith whitespace\n~~~

\n", - - "~~\nonly two\n~~\n", - "

~~\nonly two\n~~

\n", - - "```` python\nextra\n````\n", - "
extra\n
\n", - - "~~~ perl\nthree to start, four to end\n~~~~\n", - "

~~~ perl\nthree to start, four to end\n~~~~

\n", - - "~~~~ perl\nfour to start, three to end\n~~~\n", - "

~~~~ perl\nfour to start, three to end\n~~~

\n", - - "~~~ bash\ntildes\n~~~\n", - "
tildes\n
\n", - - "``` lisp\nno ending\n", - "

``` lisp\nno ending

\n", - - "~~~ lisp\nend with language\n~~~ lisp\n", - "

~~~ lisp\nend with language\n~~~ lisp

\n", - - "```\nmismatched begin and end\n~~~\n", - "

```\nmismatched begin and end\n~~~

\n", - - "~~~\nmismatched begin and end\n```\n", - "

~~~\nmismatched begin and end\n```

\n", - - " ``` oz\nleading spaces\n```\n", - "
leading spaces\n
\n", - - " ``` oz\nleading spaces\n ```\n", - "
leading spaces\n
\n", - - " ``` oz\nleading spaces\n ```\n", - "
leading spaces\n
\n", - - "``` oz\nleading spaces\n ```\n", - "
leading spaces\n
\n", - - " ``` oz\nleading spaces\n ```\n", - "
``` oz\n
\n\n

leading spaces\n ```

\n", - - "Bla bla\n\n``` oz\ncode blocks breakup paragraphs\n```\n\nBla Bla\n", - "

Bla bla

\n\n
code blocks breakup paragraphs\n
\n\n

Bla Bla

\n", - - "Some text before a fenced code block\n``` oz\ncode blocks breakup paragraphs\n```\nAnd some text after a fenced code block", - "

Some text before a fenced code block

\n\n
code blocks breakup paragraphs\n
\n\n

And some text after a fenced code block

\n", - - "`", - "

`

\n", - - "Bla bla\n\n``` oz\ncode blocks breakup paragraphs\n```\n\nBla Bla\n\n``` oz\nmultiple code blocks work okay\n```\n\nBla Bla\n", - "

Bla bla

\n\n
code blocks breakup paragraphs\n
\n\n

Bla Bla

\n\n
multiple code blocks work okay\n
\n\n

Bla Bla

\n", - - "Some text before a fenced code block\n``` oz\ncode blocks breakup paragraphs\n```\nSome text in between\n``` oz\nmultiple code blocks work okay\n```\nAnd some text after a fenced code block", - "

Some text before a fenced code block

\n\n
code blocks breakup paragraphs\n
\n\n

Some text in between

\n\n
multiple code blocks work okay\n
\n\n

And some text after a fenced code block

\n", - } - doTestsBlock(t, tests, EXTENSION_FENCED_CODE) -} - -func TestFencedCodeInsideBlockquotes(t *testing.T) { - cat := func(s ...string) string { return strings.Join(s, "\n") } - var tests = []string{ - cat("> ```go", - "package moo", - "", - "```", - ""), - `
-
package moo
-
-
-
-`, - // ------------------------------------------- - cat("> foo", - "> ", - "> ```go", - "package moo", - "```", - "> ", - "> goo.", - ""), - `
-

foo

- -
package moo
-
- -

goo.

-
-`, - // ------------------------------------------- - cat("> foo", - "> ", - "> quote", - "continues", - "```", - ""), - `
-

foo

- -

quote -continues -` + "```" + `

-
-`, - // ------------------------------------------- - cat("> foo", - "> ", - "> ```go", - "package moo", - "```", - "> ", - "> goo.", - "> ", - "> ```go", - "package zoo", - "```", - "> ", - "> woo.", - ""), - `
-

foo

- -
package moo
-
- -

goo.

- -
package zoo
-
- -

woo.

-
-`, - } - - // These 2 alternative forms of blockquoted fenced code blocks should produce same output. - forms := [2]string{ - cat("> plain quoted text", - "> ```fenced", - "code", - " with leading single space correctly preserved", - "okay", - "```", - "> rest of quoted text"), - cat("> plain quoted text", - "> ```fenced", - "> code", - "> with leading single space correctly preserved", - "> okay", - "> ```", - "> rest of quoted text"), - } - want := `
-

plain quoted text

- -
code
- with leading single space correctly preserved
-okay
-
- -

rest of quoted text

-
-` - tests = append(tests, forms[0], want) - tests = append(tests, forms[1], want) - - doTestsBlock(t, tests, EXTENSION_FENCED_CODE) -} - -func TestTable(t *testing.T) { - var tests = []string{ - "a | b\n---|---\nc | d\n", - "\n\n\n\n\n\n\n\n" + - "\n\n\n\n\n\n
ab
cd
\n", - - "a | b\n---|--\nc | d\n", - "

a | b\n---|--\nc | d

\n", - - "|a|b|c|d|\n|----|----|----|---|\n|e|f|g|h|\n", - "\n\n\n\n\n\n\n\n\n\n" + - "\n\n\n\n\n\n\n\n
abcd
efgh
\n", - - "*a*|__b__|[c](C)|d\n---|---|---|---\ne|f|g|h\n", - "\n\n\n\n\n\n\n\n\n\n" + - "\n\n\n\n\n\n\n\n
abcd
efgh
\n", - - "a|b|c\n---|---|---\nd|e|f\ng|h\ni|j|k|l|m\nn|o|p\n", - "\n\n\n\n\n\n\n\n\n" + - "\n\n\n\n\n\n\n" + - "\n\n\n\n\n\n" + - "\n\n\n\n\n\n" + - "\n\n\n\n\n\n
abc
def
gh
ijk
nop
\n", - - "a|b|c\n---|---|---\n*d*|__e__|f\n", - "\n\n\n\n\n\n\n\n\n" + - "\n\n\n\n\n\n\n
abc
def
\n", - - "a|b|c|d\n:--|--:|:-:|---\ne|f|g|h\n", - "\n\n\n\n\n" + - "\n\n\n\n\n" + - "\n\n\n\n" + - "\n\n\n\n
abcd
efgh
\n", - - "a|b|c\n---|---|---\n", - "\n\n\n\n\n\n\n\n\n\n\n
abc
\n", - - "a| b|c | d | e\n---|---|---|---|---\nf| g|h | i |j\n", - "\n\n\n\n\n\n\n\n\n\n\n" + - "\n\n\n\n\n\n\n\n\n
abcde
fghij
\n", - - "a|b\\|c|d\n---|---|---\nf|g\\|h|i\n", - "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ab|cd
fg|hi
\n", - } - doTestsBlock(t, tests, EXTENSION_TABLES) -} - -func TestUnorderedListWith_EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK(t *testing.T) { - var tests = []string{ - "* Hello\n", - "
    \n
  • Hello
    • \n
\n", - - "* Yin\n* Yang\n", - "
    \n
  • Yin
    • \n
  • Yang
    • \n
\n", - - "* Ting\n* Bong\n* Goo\n", - "
    \n
  • Ting
    • \n
  • Bong
    • \n
  • Goo
    • \n
\n", - - "* Yin\n\n* Yang\n", - "
    \n

  • Yin

    • \n\n

  • Yang

    • \n
\n", - - "* Ting\n\n* Bong\n* Goo\n", - "
    \n

  • Ting

    • \n\n

  • Bong

    • \n\n

  • Goo

    • \n
\n", - - "+ Hello\n", - "
    \n
  • Hello
    • \n
\n", - - "+ Yin\n+ Yang\n", - "
    \n
  • Yin
    • \n
  • Yang
    • \n
\n", - - "+ Ting\n+ Bong\n+ Goo\n", - "
    \n
  • Ting
    • \n
  • Bong
    • \n
  • Goo
    • \n
\n", - - "+ Yin\n\n+ Yang\n", - "
    \n

  • Yin

    • \n\n

  • Yang

    • \n
\n", - - "+ Ting\n\n+ Bong\n+ Goo\n", - "
    \n

  • Ting

    • \n\n

  • Bong

    • \n\n

  • Goo

    • \n
\n", - - "- Hello\n", - "
    \n
  • Hello
    • \n
\n", - - "- Yin\n- Yang\n", - "
    \n
  • Yin
    • \n
  • Yang
    • \n
\n", - - "- Ting\n- Bong\n- Goo\n", - "
    \n
  • Ting
    • \n
  • Bong
    • \n
  • Goo
    • \n
\n", - - "- Yin\n\n- Yang\n", - "
    \n

  • Yin

    • \n\n

  • Yang

    • \n
\n", - - "- Ting\n\n- Bong\n- Goo\n", - "
    \n

  • Ting

    • \n\n

  • Bong

    • \n\n

  • Goo

    • \n
\n", - - "*Hello\n", - "

*Hello

\n", - - "* Hello \n", - "
    \n
  • Hello
    • \n
\n", - - "* Hello \n Next line \n", - "
    \n
  • Hello\nNext line
    • \n
\n", - - "Paragraph\n* No linebreak\n", - "

Paragraph

\n\n
    \n
  • No linebreak
    • \n
\n", - - "Paragraph\n\n* Linebreak\n", - "

Paragraph

\n\n
    \n
  • Linebreak
    • \n
\n", - - "* List\n * Nested list\n", - "
    \n
  • List\n\n
      \n
    • Nested list
      • \n
    • \n
\n", - - "* List\n\n * Nested list\n", - "
    \n

  • List

    \n\n
      \n
    • Nested list
      • \n
    • \n
\n", - - "* List\n Second line\n\n + Nested\n", - "
    \n

  • List\nSecond line

    \n\n
      \n
    • Nested
      • \n
    • \n
\n", - - "* List\n + Nested\n\n Continued\n", - "
    \n

  • List

    \n\n
      \n
    • Nested
      • \n
    \n\n

    Continued

    • \n
\n", - - "* List\n * shallow indent\n", - "
    \n
  • List\n\n
      \n
    • shallow indent
      • \n
    • \n
\n", - - "* List\n" + - " * shallow indent\n" + - " * part of second list\n" + - " * still second\n" + - " * almost there\n" + - " * third level\n", - "
    \n" + - "
  • List\n\n" + - "
      \n" + - "
    • shallow indent
      • \n" + - "
    • part of second list
      • \n" + - "
    • still second
      • \n" + - "
    • almost there\n\n" + - "
        \n" + - "
      • third level
        • \n" + - "
      • \n" + - "
    • \n" + - "
\n", - - "* List\n extra indent, same paragraph\n", - "
    \n
  • List\n extra indent, same paragraph
    • \n
\n", - - "* List\n\n code block\n", - "
    \n

  • List

    \n\n
    code block\n
    • \n
\n", - - "* List\n\n code block with spaces\n", - "
    \n

  • List

    \n\n
      code block with spaces\n
    • \n
\n", - - "* List\n\n * sublist\n\n normal text\n\n * another sublist\n", - "
    \n

  • List

    \n\n
      \n
    • sublist
      • \n
    \n\n

    normal text

    \n\n
      \n
    • another sublist
      • \n
    • \n
\n", - } - doTestsBlock(t, tests, EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK) -} - -func TestOrderedList_EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK(t *testing.T) { - var tests = []string{ - "1. Hello\n", - "
    \n
  1. Hello
    2. \n
\n", - - "1. Yin\n2. Yang\n", - "
    \n
  1. Yin
    2. \n
  2. Yang
    3. \n
\n", - - "1. Ting\n2. Bong\n3. Goo\n", - "
    \n
  1. Ting
    2. \n
  2. Bong
    3. \n
  3. Goo
    4. \n
\n", - - "1. Yin\n\n2. Yang\n", - "
    \n

  1. Yin

    2. \n\n

  2. Yang

    3. \n
\n", - - "1. Ting\n\n2. Bong\n3. Goo\n", - "
    \n

  1. Ting

    2. \n\n

  2. Bong

    3. \n\n

  3. Goo

    4. \n
\n", - - "1 Hello\n", - "

1 Hello

\n", - - "1.Hello\n", - "

1.Hello

\n", - - "1. Hello \n", - "
    \n
  1. Hello
    2. \n
\n", - - "1. Hello \n Next line \n", - "
    \n
  1. Hello\nNext line
    2. \n
\n", - - "Paragraph\n1. No linebreak\n", - "

Paragraph

\n\n
    \n
  1. No linebreak
    2. \n
\n", - - "Paragraph\n\n1. Linebreak\n", - "

Paragraph

\n\n
    \n
  1. Linebreak
    2. \n
\n", - - "1. List\n 1. Nested list\n", - "
    \n
  1. List\n\n
      \n
    1. Nested list
      2. \n
    2. \n
\n", - - "1. List\n\n 1. Nested list\n", - "
    \n

  1. List

    \n\n
      \n
    1. Nested list
      2. \n
    2. \n
\n", - - "1. List\n Second line\n\n 1. Nested\n", - "
    \n

  1. List\nSecond line

    \n\n
      \n
    1. Nested
      2. \n
    2. \n
\n", - - "1. List\n 1. Nested\n\n Continued\n", - "
    \n

  1. List

    \n\n
      \n
    1. Nested
      2. \n
    \n\n

    Continued

    2. \n
\n", - - "1. List\n 1. shallow indent\n", - "
    \n
  1. List\n\n
      \n
    1. shallow indent
      2. \n
    2. \n
\n", - - "1. List\n" + - " 1. shallow indent\n" + - " 2. part of second list\n" + - " 3. still second\n" + - " 4. almost there\n" + - " 1. third level\n", - "
    \n" + - "
  1. List\n\n" + - "
      \n" + - "
    1. shallow indent
      2. \n" + - "
    2. part of second list
      3. \n" + - "
    3. still second
      4. \n" + - "
    4. almost there\n\n" + - "
        \n" + - "
      1. third level
        2. \n" + - "
      5. \n" + - "
    2. \n" + - "
\n", - - "1. List\n extra indent, same paragraph\n", - "
    \n
  1. List\n extra indent, same paragraph
    2. \n
\n", - - "1. List\n\n code block\n", - "
    \n

  1. List

    \n\n
    code block\n
    2. \n
\n", - - "1. List\n\n code block with spaces\n", - "
    \n

  1. List

    \n\n
      code block with spaces\n
    2. \n
\n", - - "1. List\n * Mixted list\n", - "
    \n
  1. List\n\n
      \n
    • Mixted list
      • \n
    2. \n
\n", - - "1. List\n * Mixed list\n", - "
    \n
  1. List\n\n
      \n
    • Mixed list
      • \n
    2. \n
\n", - - "* Start with unordered\n 1. Ordered\n", - "
    \n
  • Start with unordered\n\n
      \n
    1. Ordered
      2. \n
    • \n
\n", - - "* Start with unordered\n 1. Ordered\n", - "
    \n
  • Start with unordered\n\n
      \n
    1. Ordered
      2. \n
    • \n
\n", - - "1. numbers\n1. are ignored\n", - "
    \n
  1. numbers
    2. \n
  2. are ignored
    3. \n
\n", - } - doTestsBlock(t, tests, EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK) -} - -func TestFencedCodeBlock_EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK(t *testing.T) { - var tests = []string{ - "``` go\nfunc foo() bool {\n\treturn true;\n}\n```\n", - "
func foo() bool {\n\treturn true;\n}\n
\n", - - "``` c\n/* special & char < > \" escaping */\n```\n", - "
/* special & char < > " escaping */\n
\n", - - "``` c\nno *inline* processing ~~of text~~\n```\n", - "
no *inline* processing ~~of text~~\n
\n", - - "```\nNo language\n```\n", - "
No language\n
\n", - - "``` {ocaml}\nlanguage in braces\n```\n", - "
language in braces\n
\n", - - "``` {ocaml} \nwith extra whitespace\n```\n", - "
with extra whitespace\n
\n", - - "```{ ocaml }\nwith extra whitespace\n```\n", - "
with extra whitespace\n
\n", - - "~ ~~ java\nWith whitespace\n~~~\n", - "

~ ~~ java\nWith whitespace\n~~~

\n", - - "~~\nonly two\n~~\n", - "

~~\nonly two\n~~

\n", - - "```` python\nextra\n````\n", - "
extra\n
\n", - - "~~~ perl\nthree to start, four to end\n~~~~\n", - "

~~~ perl\nthree to start, four to end\n~~~~

\n", - - "~~~~ perl\nfour to start, three to end\n~~~\n", - "

~~~~ perl\nfour to start, three to end\n~~~

\n", - - "~~~ bash\ntildes\n~~~\n", - "
tildes\n
\n", - - "``` lisp\nno ending\n", - "

``` lisp\nno ending

\n", - - "~~~ lisp\nend with language\n~~~ lisp\n", - "

~~~ lisp\nend with language\n~~~ lisp

\n", - - "```\nmismatched begin and end\n~~~\n", - "

```\nmismatched begin and end\n~~~

\n", - - "~~~\nmismatched begin and end\n```\n", - "

~~~\nmismatched begin and end\n```

\n", - - " ``` oz\nleading spaces\n```\n", - "
leading spaces\n
\n", - - " ``` oz\nleading spaces\n ```\n", - "
leading spaces\n
\n", - - " ``` oz\nleading spaces\n ```\n", - "
leading spaces\n
\n", - - "``` oz\nleading spaces\n ```\n", - "
leading spaces\n
\n", - - " ``` oz\nleading spaces\n ```\n", - "
``` oz\n
\n\n

leading spaces

\n\n
```\n
\n", - } - doTestsBlock(t, tests, EXTENSION_FENCED_CODE|EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK) -} - -func TestTitleBlock_EXTENSION_TITLEBLOCK(t *testing.T) { - var tests = []string{ - "% Some title\n" + - "% Another title line\n" + - "% Yep, more here too\n", - "

" + - "Some title\n" + - "Another title line\n" + - "Yep, more here too\n" + - "

", - } - doTestsBlock(t, tests, EXTENSION_TITLEBLOCK) -} - -func TestBlockComments(t *testing.T) { - var tests = []string{ - "Some text\n\n\n", - "

Some text

\n\n\n", - - "Some text\n\n\n", - "

Some text

\n\n\n", - - "Some text\n\n\n", - "

Some text

\n\n\n", - } - doTestsBlock(t, tests, 0) -} - -func TestCDATA(t *testing.T) { - var tests = []string{ - "Some text\n\n\n", - "

Some text

\n\n\n", - - "CDATA ]]\n\n\n", - "

CDATA ]]

\n\n\n", - - "CDATA >\n\n]]>\n", - "

CDATA >

\n\n]]>\n", - - "Lots of text\n\n\n", - "

Lots of text

\n\n\n", - - "]]>\n", - "]]>\n", - } - doTestsBlock(t, tests, 0) - doTestsBlock(t, []string{ - "``` html\n\n```\n", - "
<![CDATA[foo]]>\n
\n", - - "\n", - "\n", - - ` def func(): -> pass -]]> -`, - ` def func(): -> pass -]]> -`, - }, EXTENSION_FENCED_CODE) -} diff --git a/vendor/github.com/russross/blackfriday/html.go b/vendor/github.com/russross/blackfriday/html.go deleted file mode 100644 index 74e67ee8..00000000 --- a/vendor/github.com/russross/blackfriday/html.go +++ /dev/null @@ -1,949 +0,0 @@ -// -// Blackfriday Markdown Processor -// Available at http://github.com/russross/blackfriday -// -// Copyright © 2011 Russ Ross . -// Distributed under the Simplified BSD License. -// See README.md for details. -// - -// -// -// HTML rendering backend -// -// - -package blackfriday - -import ( - "bytes" - "fmt" - "regexp" - "strconv" - "strings" -) - -// Html renderer configuration options. -const ( - HTML_SKIP_HTML = 1 << iota // skip preformatted HTML blocks - HTML_SKIP_STYLE // skip embedded